diff --git a/Cargo.lock b/Cargo.lock index 7db06f15..5a4b171e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -690,9 +690,9 @@ dependencies = [ [[package]] name = "proc-macro2" -version = "1.0.79" +version = "1.0.86" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e835ff2298f5721608eb1a980ecaee1aef2c132bf95ecc026a11b7bf3c01c02e" +checksum = "5e719e8df665df0d1c8fbfd238015744736151d4445ec0836b8e628aae103b77" dependencies = [ "unicode-ident", ] @@ -725,9 +725,9 @@ checksum = "a1d01941d82fa2ab50be1e79e6714289dd7cde78eba4c074bc5a4374f650dfe0" [[package]] name = "quote" -version = "1.0.35" +version = "1.0.36" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "291ec9ab5efd934aaf503a6466c5d5251535d108ee747472c3977cc5acc868ef" +checksum = "0fa76aaf39101c457836aec0ce2316dbdc3ab723cdda1c6bd4e6ad4208acaca7" dependencies = [ "proc-macro2", ] @@ -1026,18 +1026,18 @@ dependencies = [ [[package]] name = "serde" -version = "1.0.197" +version = "1.0.203" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3fb1c873e1b9b056a4dc4c0c198b24c3ffa059243875552b2bd0933b1aee4ce2" +checksum = "7253ab4de971e72fb7be983802300c30b5a7f0c2e56fab8abfc6a214307c0094" dependencies = [ "serde_derive", ] [[package]] name = "serde_derive" -version = "1.0.197" +version = "1.0.203" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7eb0b34b42edc17f6b7cac84a52a1c5f0e1bb2227e997ca9011ea3dd34e8610b" +checksum = "500cbc0ebeb6f46627f50f3f5811ccf6bf00643be300b4c3eabc0ef55dc5b5ba" dependencies = [ "proc-macro2", "quote", @@ -1236,9 +1236,9 @@ dependencies = [ [[package]] name = "time" -version = "0.3.34" +version = "0.3.36" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c8248b6521bb14bc45b4067159b9b6ad792e2d6d754d6c41fb50e29fefe38749" +checksum = "5dfd88e563464686c916c7e46e623e520ddc6d79fa6641390f2e3fa86e83e885" dependencies = [ "deranged", "itoa", @@ -1259,9 +1259,9 @@ checksum = "ef927ca75afb808a4d64dd374f00a2adf8d0fcff8e7b184af886c3c87ec4a3f3" [[package]] name = "time-macros" -version = "0.2.17" +version = "0.2.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7ba3a3ef41e6672a2f0f001392bb5dcd3ff0a9992d618ca761a11c3121547774" +checksum = "3f252a68540fde3a3877aeea552b832b40ab9a69e318efd078774a01ddee1ccf" dependencies = [ "num-conv", "time-core", diff --git a/Justfile b/Justfile index ee5c2790..aedb94dc 100644 --- a/Justfile +++ b/Justfile @@ -47,6 +47,9 @@ build-bin profile="dev": (build-lib profile) build-lib profile="dev": cargo build --package rsonpath-lib --profile {{profile}} +build-avx512 profile="dev": + rustup run nightly cargo build --package rsonpath-lib --profile dev + # Build all rsonpath parts, the binary and library. build-all profile="dev": (build-lib profile) (build-bin profile) (gen-tests) diff --git a/crates/rsonpath-lib/Cargo.toml b/crates/rsonpath-lib/Cargo.toml index d2f5ead4..ef3896f3 100644 --- a/crates/rsonpath-lib/Cargo.toml +++ b/crates/rsonpath-lib/Cargo.toml @@ -49,6 +49,7 @@ default = ["simd"] arbitrary = ["dep:arbitrary"] simd = [] + [[example]] name = "approx_spans_usage" path = "examples/approx_spans_usage.rs" diff --git a/crates/rsonpath-lib/src/classification/depth.rs b/crates/rsonpath-lib/src/classification/depth.rs index 43b3aaea..a20346b0 100644 --- a/crates/rsonpath-lib/src/classification/depth.rs +++ b/crates/rsonpath-lib/src/classification/depth.rs @@ -75,10 +75,15 @@ pub(crate) mod shared; pub(crate) mod avx2_32; #[cfg(target_arch = "x86_64")] pub(crate) mod avx2_64; +#[cfg(target_arch = "x86_64")] +pub(crate) mod avx512_64; #[cfg(target_arch = "x86")] pub(crate) mod sse2_32; #[cfg(target_arch = "x86_64")] pub(crate) mod sse2_64; +#[cfg(target_arch = "aarch64")] +pub(crate) mod neon_64; + pub(crate) trait DepthImpl { type Classifier<'i, I, Q>: DepthIterator<'i, I, Q, MaskType, BLOCK_SIZE> diff --git a/crates/rsonpath-lib/src/classification/depth/avx512_64.rs b/crates/rsonpath-lib/src/classification/depth/avx512_64.rs new file mode 100644 index 00000000..27cdd041 --- /dev/null +++ b/crates/rsonpath-lib/src/classification/depth/avx512_64.rs @@ -0,0 +1,56 @@ +use super::{ + shared::{mask_64::DepthVector64, vector_512::DelimiterClassifierImpl512}, + *, +}; +use crate::{ + classification::{QuoteClassifiedBlock, ResumeClassifierBlockState}, + debug, + input::InputBlock, +}; +use std::marker::PhantomData; + +const SIZE: usize = 64; + +shared::depth_classifier!(Avx512VectorIterator64, DelimiterClassifierImpl512, DepthVector64, 64, u64); + +#[inline(always)] +fn new_vector<'a, B: InputBlock<'a, SIZE>>( + bytes: QuoteClassifiedBlock, + classifier: &DelimiterClassifierImpl512, +) -> DepthVector64<'a, B> { + new_vector_from(bytes, classifier, 0) +} + +#[inline(always)] +fn new_vector_from<'a, B: InputBlock<'a, SIZE>>( + bytes: QuoteClassifiedBlock, + classifier: &DelimiterClassifierImpl512, + idx: usize, +) -> DepthVector64<'a, B> { + // SAFETY: target_feature invariant + unsafe { new_avx512(bytes, classifier, idx) } +} + +#[inline(always)] +unsafe fn new_avx512<'a, B: InputBlock<'a, SIZE>>( + bytes: QuoteClassifiedBlock, + classifier: &DelimiterClassifierImpl512, + start_idx: usize, +) -> DepthVector64<'a, B> { + let idx_mask = 0xFFFF_FFFF_FFFF_FFFF_u64 << start_idx; + let block = &bytes.block; + let (opening_mask, closing_mask) = classifier.get_opening_and_closing_masks(block); + + let opening_mask = opening_mask & (!bytes.within_quotes_mask) & idx_mask; + let closing_mask = closing_mask & (!bytes.within_quotes_mask) & idx_mask; + + DepthVector64 { + quote_classified: bytes, + opening_mask, + closing_mask, + opening_count: opening_mask.count_ones(), + depth: 0, + idx: 0, + phantom: PhantomData, + } +} diff --git a/crates/rsonpath-lib/src/classification/depth/neon_64.rs b/crates/rsonpath-lib/src/classification/depth/neon_64.rs new file mode 100644 index 00000000..552d13f5 --- /dev/null +++ b/crates/rsonpath-lib/src/classification/depth/neon_64.rs @@ -0,0 +1,62 @@ +use super::{ + shared::{mask_neon::DepthVectorNeon, vector_neon::DelimiterClassifierImplNeon}, + *, +}; +use crate::{ + classification::{mask::m64, QuoteClassifiedBlock, ResumeClassifierBlockState}, + debug, + input::InputBlock, +}; +use std::marker::PhantomData; + +const SIZE: usize = 64; + +shared::depth_classifier!(NeonVectorIterator, DelimiterClassifierImplNeon, DepthVectorNeon, 64, u64); + +#[inline(always)] +fn new_vector<'a, B: InputBlock<'a, SIZE>>( + bytes: QuoteClassifiedBlock, + classifier: &DelimiterClassifierImplNeon, +) -> DepthVectorNeon<'a, B> { + new_vector_from(bytes, classifier, 0) +} + +#[inline(always)] +fn new_vector_from<'a, B: InputBlock<'a, SIZE>>( + bytes: QuoteClassifiedBlock, + classifier: &DelimiterClassifierImplNeon, + idx: usize, +) -> DepthVectorNeon<'a, B> { + // SAFETY: target_feature invariant + unsafe { new_neon(bytes, classifier, idx) } +} + +#[inline(always)] +unsafe fn new_neon<'a, B: InputBlock<'a, SIZE>>( + bytes: QuoteClassifiedBlock, + classifier: &DelimiterClassifierImplNeon, + start_idx: usize, +) -> DepthVectorNeon<'a, B> { + let idx_mask = 0xFFFF_FFFF_FFFF_FFFF_u64 << start_idx; + let (block1, block2, block3, block4) = bytes.block.quarters(); + let (opening_mask1, closing_mask1) = classifier.get_opening_and_closing_masks(block1); + let (opening_mask2, closing_mask2) = classifier.get_opening_and_closing_masks(block2); + let (opening_mask3, closing_mask3) = classifier.get_opening_and_closing_masks(block3); + let (opening_mask4, closing_mask4) = classifier.get_opening_and_closing_masks(block4); + + let combined_opening_mask = m64::combine_16(opening_mask1, opening_mask2, opening_mask3, opening_mask4); + let combined_closing_mask = m64::combine_16(closing_mask1, closing_mask2, closing_mask3, closing_mask4); + + let opening_mask = combined_opening_mask & (!bytes.within_quotes_mask) & idx_mask; + let closing_mask = combined_closing_mask & (!bytes.within_quotes_mask) & idx_mask; + + DepthVectorNeon { + quote_classified: bytes, + opening_mask, + closing_mask, + opening_count: opening_mask.count_ones(), + depth: 0, + idx: 0, + phantom: PhantomData, + } +} diff --git a/crates/rsonpath-lib/src/classification/depth/shared.rs b/crates/rsonpath-lib/src/classification/depth/shared.rs index cd639507..c053e7ef 100644 --- a/crates/rsonpath-lib/src/classification/depth/shared.rs +++ b/crates/rsonpath-lib/src/classification/depth/shared.rs @@ -2,10 +2,16 @@ pub(super) mod mask_32; #[cfg(target_arch = "x86_64")] pub(super) mod mask_64; +#[cfg(target_arch = "aarch64")] +pub(super) mod mask_neon; #[cfg(any(target_arch = "x86", target_arch = "x86_64"))] pub(super) mod vector_128; #[cfg(any(target_arch = "x86", target_arch = "x86_64"))] pub(super) mod vector_256; +#[cfg(any(target_arch = "x86", target_arch = "x86_64"))] +pub(super) mod vector_512; +#[cfg(target_arch = "aarch64")] +pub(super) mod vector_neon; #[allow(unused_macros)] macro_rules! depth_classifier { diff --git a/crates/rsonpath-lib/src/classification/depth/shared/mask_neon.rs b/crates/rsonpath-lib/src/classification/depth/shared/mask_neon.rs new file mode 100644 index 00000000..7b2171d3 --- /dev/null +++ b/crates/rsonpath-lib/src/classification/depth/shared/mask_neon.rs @@ -0,0 +1,84 @@ +use crate::{ + bin_u64, + classification::{depth::DepthBlock, quotes::QuoteClassifiedBlock}, + debug, + input::InputBlock, +}; +use std::marker::PhantomData; + +const SIZE: usize = 64; + +/// Works on a 64-byte slice, but uses a heuristic to quickly +/// respond to queries and not count the depth exactly unless +/// needed. +/// +/// The heuristic checks if it is possible to achieve the queried +/// depth within the block by counting the number of opening +/// and closing structural characters. This can be done much +/// more quickly than precise depth calculation. +pub(crate) struct DepthVectorNeon<'a, B: InputBlock<'a, SIZE>> { + pub(crate) quote_classified: QuoteClassifiedBlock, + pub(crate) opening_mask: u64, + pub(crate) opening_count: u32, + pub(crate) closing_mask: u64, + pub(crate) idx: usize, + pub(crate) depth: i32, + pub(crate) phantom: PhantomData<&'a ()>, +} + +// TODO FIXME: consider rewriting training and count_zeros etc. functions. + +impl<'a, B: InputBlock<'a, SIZE>> DepthBlock<'a> for DepthVectorNeon<'a, B> { + #[inline(always)] + fn advance_to_next_depth_decrease(&mut self) -> bool { + let next_closing = self.closing_mask.trailing_zeros() as usize; + + if next_closing == SIZE { + return false; + } + + bin_u64!("opening_mask", self.opening_mask); + bin_u64!("closing_mask", self.closing_mask); + + self.opening_mask >>= next_closing; + self.closing_mask >>= next_closing; + self.opening_mask >>= 1; + self.closing_mask >>= 1; + + bin_u64!("new opening_mask", self.opening_mask); + bin_u64!("new closing_mask", self.closing_mask); + + let new_opening_count = self.opening_mask.count_ones() as i32; + let delta = (self.opening_count as i32) - new_opening_count - 1; + self.opening_count = new_opening_count as u32; + + debug!("next_closing: {next_closing}"); + debug!("new_opening_count: {new_opening_count}"); + debug!("delta: {delta}"); + + self.depth += delta; + self.idx += next_closing + 1; + + true + } + + #[inline(always)] + fn get_depth(&self) -> isize { + self.depth as isize + } + + #[inline(always)] + fn depth_at_end(&self) -> isize { + (((self.opening_count as i32) - self.closing_mask.count_ones() as i32) + self.depth) as isize + } + + #[inline(always)] + fn add_depth(&mut self, depth: isize) { + self.depth += depth as i32; + } + + #[inline(always)] + fn estimate_lowest_possible_depth(&self) -> isize { + (self.depth - self.closing_mask.count_ones() as i32) as isize + } +} diff --git a/crates/rsonpath-lib/src/classification/depth/shared/vector_512.rs b/crates/rsonpath-lib/src/classification/depth/shared/vector_512.rs new file mode 100644 index 00000000..2dfc8281 --- /dev/null +++ b/crates/rsonpath-lib/src/classification/depth/shared/vector_512.rs @@ -0,0 +1,46 @@ +use crate::classification::structural::BracketType; + +#[cfg(target_arch = "x86")] +use core::arch::x86::*; +#[cfg(target_arch = "x86_64")] +use core::arch::x86_64::*; + +pub(crate) struct DelimiterClassifierImpl512 { + opening: i8, +} + +impl DelimiterClassifierImpl512 { + pub(crate) fn new(opening: BracketType) -> Self { + let opening = match opening { + BracketType::Square => b'[', + BracketType::Curly => b'{', + }; + + Self { opening: opening as i8 } + } + + #[inline(always)] + unsafe fn opening_mask(&self) -> __m512i { + _mm512_set1_epi8(self.opening) + } + + #[inline(always)] + unsafe fn closing_mask(&self) -> __m512i { + _mm512_set1_epi8(self.opening + 2) + } + + #[target_feature(enable = "avx512f")] + #[target_feature(enable = "avx512bw")] + #[inline] + pub(crate) unsafe fn get_opening_and_closing_masks(&self, bytes: &[u8]) -> (u64, u64) { + assert_eq!(64, bytes.len()); + // SAFETY: target_feature invariant + unsafe { + let byte_vector = _mm512_loadu_si512(bytes.as_ptr().cast::()); + let opening_mask = _mm512_cmpeq_epi8_mask(byte_vector, self.opening_mask()); + let closing_mask = _mm512_cmpeq_epi8_mask(byte_vector, self.closing_mask()); + + (opening_mask, closing_mask) + } + } +} diff --git a/crates/rsonpath-lib/src/classification/depth/shared/vector_neon.rs b/crates/rsonpath-lib/src/classification/depth/shared/vector_neon.rs new file mode 100644 index 00000000..7af8f79e --- /dev/null +++ b/crates/rsonpath-lib/src/classification/depth/shared/vector_neon.rs @@ -0,0 +1,67 @@ +use crate::classification::structural::BracketType; + +#[cfg(target_arch = "aarch64")] +use ::core::arch::aarch64::*; + +#[inline] +#[target_feature(enable = "neon")] +unsafe fn neon_movemask_epi8(cmp_vector: uint8x16_t) -> i16 { + // Tablica shiftów + let shift_values: [i8; 16] = [-7, -6, -5, -4, -3, -2, -1, 0, -7, -6, -5, -4, -3, -2, -1, 0]; + + // Załaduj shift do NEON + let vshift = vld1q_s8(shift_values.as_ptr()); + + // Zamaskowanie bitu 7 (0x80) + let vmask = vandq_u8(cmp_vector, vdupq_n_u8(0x80)); + + // Przesunięcie bitów + let vmask = vshlq_u8(vmask, vshift); + + // Sumowanie dolnej i górnej połowy + let low = vaddv_u8(vget_low_u8(vmask)) as i16; + let high = vaddv_u8(vget_high_u8(vmask)) as i16; + + low | (high << 8) +} + +pub(crate) struct DelimiterClassifierImplNeon { + opening: i8, +} + +impl DelimiterClassifierImplNeon { + pub(crate) fn new(opening: BracketType) -> Self { + let opening = match opening { + BracketType::Square => b'[', + BracketType::Curly => b'{', + }; + + Self { opening: opening as i8 } + } + + #[inline(always)] + unsafe fn opening_mask(&self) -> int8x16_t { + vdupq_n_s8(self.opening) + } + + #[inline(always)] + unsafe fn closing_mask(&self) -> int8x16_t { + vdupq_n_s8(self.opening + 2) + } + + #[target_feature(enable = "neon")] + #[inline] + pub(crate) unsafe fn get_opening_and_closing_masks(&self, bytes: &[u8]) -> (u16, u16) { + assert_eq!(16, bytes.len()); + // SAFETY: target_feature invariant + unsafe { + let byte_vector = vreinterpretq_s8_u8(vld1q_u8(bytes.as_ptr())); + let opening_brace_cmp = vceqq_s8(byte_vector, self.opening_mask()); + let closing_brace_cmp = vceqq_s8(byte_vector, self.closing_mask()); + let opening_mask = neon_movemask_epi8(opening_brace_cmp) as u16; + let closing_mask = neon_movemask_epi8(closing_brace_cmp) as u16; + + (opening_mask, closing_mask) + } + } +} diff --git a/crates/rsonpath-lib/src/classification/mask.rs b/crates/rsonpath-lib/src/classification/mask.rs index 6781d334..45015db4 100644 --- a/crates/rsonpath-lib/src/classification/mask.rs +++ b/crates/rsonpath-lib/src/classification/mask.rs @@ -6,6 +6,16 @@ pub(crate) trait Mask { Self: Shl; } +impl Mask for u128 { + #[inline(always)] + fn is_lit(&self, bit: N) -> bool + where + Self: Shl, + { + (*self & (1 << bit)) != 0 + } +} + impl Mask for u64 { #[inline(always)] fn is_lit(&self, bit: N) -> bool @@ -53,3 +63,20 @@ pub(crate) mod m64 { u64::from(m1) | (u64::from(m2) << 32) } } + +#[allow(dead_code)] +pub(crate) mod m128 { + pub(crate) fn combine_16(m1: u16, m2: u16, m3: u16, m4: u16, + m5: u16, m6: u16, m7: u16, m8: u16) -> u128 { + u128::from(m1) | (u128::from(m2) << 16) | (u128::from(m3) << 32) | (u128::from(m4) << 48) | + (u128::from(m5) << 64) | (u128::from(m6) << 80) | (u128::from(m7) << 96) | (u128::from(m8) << 112) + } + + pub(crate) fn combine_32(m1: u32, m2: u32, m3: u32, m4: u32) -> u128 { + u128::from(m1) | (u128::from(m2) << 32) | (u128::from(m3) << 64) | (u128::from(m4) << 96) + } + + pub(crate) fn combine_64(m1: u64, m2: u64) -> u128 { + u128::from(m1) | (u128::from(m2) << 64) + } +} diff --git a/crates/rsonpath-lib/src/classification/memmem.rs b/crates/rsonpath-lib/src/classification/memmem.rs index b8b872d2..1a0e08ea 100644 --- a/crates/rsonpath-lib/src/classification/memmem.rs +++ b/crates/rsonpath-lib/src/classification/memmem.rs @@ -33,10 +33,14 @@ pub(crate) mod shared; pub(crate) mod avx2_32; #[cfg(target_arch = "x86_64")] pub(crate) mod avx2_64; +#[cfg(target_arch = "x86_64")] +pub(crate) mod avx512_64; #[cfg(target_arch = "x86")] pub(crate) mod sse2_32; #[cfg(target_arch = "x86_64")] pub(crate) mod sse2_64; +#[cfg(target_arch = "aarch64")] +pub(crate) mod neon_64; pub(crate) trait MemmemImpl { type Classifier<'i, 'b, 'r, I, R>: Memmem<'i, 'b, 'r, I, BLOCK_SIZE> diff --git a/crates/rsonpath-lib/src/classification/memmem/avx2_64.rs b/crates/rsonpath-lib/src/classification/memmem/avx2_64.rs index b847885f..54807e01 100644 --- a/crates/rsonpath-lib/src/classification/memmem/avx2_64.rs +++ b/crates/rsonpath-lib/src/classification/memmem/avx2_64.rs @@ -135,7 +135,6 @@ where let classifier = vector_256::BlockClassifier256::new(label.unquoted().as_bytes()[0], label.unquoted().as_bytes()[1]); let mut previous_block: u64 = 0; - while let Some(block) = self.iter.next().e()? { let (block1, block2) = block.halves(); let classified1 = classifier.classify_block(block1); @@ -149,11 +148,9 @@ where { return Ok(Some((res, block))); } - offset += SIZE; previous_block = first_bitmask >> (SIZE - 1); } - Ok(None) } } diff --git a/crates/rsonpath-lib/src/classification/memmem/avx512_64.rs b/crates/rsonpath-lib/src/classification/memmem/avx512_64.rs new file mode 100644 index 00000000..8c72d892 --- /dev/null +++ b/crates/rsonpath-lib/src/classification/memmem/avx512_64.rs @@ -0,0 +1,175 @@ +use super::{shared::mask_64, shared::vector_512, *}; +use crate::{ + input::{error::InputErrorConvertible, InputBlockIterator}, +}; + +const SIZE: usize = 64; + +pub(crate) struct Constructor; + +impl MemmemImpl for Constructor { + type Classifier<'i, 'b, 'r, I, R> = Avx512MemmemClassifier64<'i, 'b, 'r, I, R> + where + I: Input + 'i, + ::BlockIterator<'i, 'r, R, BLOCK_SIZE>: 'b, + R: InputRecorder<::Block<'i, BLOCK_SIZE>> + 'r, + 'i: 'r; + + fn memmem<'i, 'b, 'r, I, R>( + input: &'i I, + iter: &'b mut ::BlockIterator<'i, 'r, R, BLOCK_SIZE>, + ) -> Self::Classifier<'i, 'b, 'r, I, R> + where + I: Input, + R: InputRecorder<::Block<'i, BLOCK_SIZE>>, + 'i: 'r, + { + Self::Classifier { input, iter } + } +} + +pub(crate) struct Avx512MemmemClassifier64<'i, 'b, 'r, I, R> +where + I: Input, + R: InputRecorder> + 'r, +{ + input: &'i I, + iter: &'b mut I::BlockIterator<'i, 'r, R, SIZE>, +} + +impl<'i, 'b, 'r, I, R> Avx512MemmemClassifier64<'i, 'b, 'r, I, R> +where + I: Input, + R: InputRecorder>, + 'i: 'r, +{ + #[inline] + #[allow(dead_code)] + pub(crate) fn new(input: &'i I, iter: &'b mut I::BlockIterator<'i, 'r, R, SIZE>) -> Self { + Self { input, iter } + } + + #[inline(always)] + unsafe fn find_empty( + &mut self, + label: &JsonString, + mut offset: usize, + ) -> Result)>, InputError> { + let classifier = vector_512::BlockClassifier512::new(b'"', b'"'); + let mut previous_block: u64 = 0; + while let Some(block) = self.iter.next().e()? { + let classified = classifier.classify_block(&block); + + let mut result = (previous_block | (classified.first << 1)) & classified.second; + while result != 0 { + let idx = result.trailing_zeros() as usize; + if self + .input + .is_member_match(offset + idx - 1, offset + idx + 1, label) + .e()? + { + return Ok(Some((offset + idx - 1, block))); + } + result &= !(1 << idx); + } + + offset += SIZE; + previous_block = classified.first >> (SIZE - 1); + } + + Ok(None) + } + + // Here we want to detect the pattern `"c"` + // For interblock communication we need to bit of information that requires extra work to get obtained. + // one for the block cut being `"` and `c"` and one for `"c` and `"`. We only deal with one of them. + #[inline(always)] + unsafe fn find_letter( + &mut self, + label: &JsonString, + mut offset: usize, + ) -> Result)>, InputError> { + let classifier = vector_512::BlockClassifier512::new(label.unquoted().as_bytes()[0], b'"'); + let mut previous_block: u64 = 0; + + while let Some(block) = self.iter.next().e()? { + let classified = classifier.classify_block(&block); + + if let Some(res) = mask_64::find_in_mask( + self.input, + label, + previous_block, + classified.first, + classified.second, + offset, + )? { + return Ok(Some((res, block))); + } + + offset += SIZE; + previous_block = classified.first >> (SIZE - 1); + } + + Ok(None) + } + + #[inline(always)] + unsafe fn find_label_avx512( + &mut self, + label: &JsonString, + mut offset: usize, + ) -> Result)>, InputError> { + if label.unquoted().is_empty() { + return self.find_empty(label, offset); + } else if label.unquoted().len() == 1 { + return self.find_letter(label, offset); + } + + let classifier = + vector_512::BlockClassifier512::new(label.unquoted().as_bytes()[0], label.unquoted().as_bytes()[1]); + let mut previous_block: u64 = 0; + while let Some(block) = self.iter.next().e()? { + let classified = classifier.classify_block(&block); + + if let Some(res) = mask_64::find_in_mask( + self.input, + label, + previous_block, + classified.first, + classified.second, + offset, + )? { + return Ok(Some((res, block))); + } + + offset += SIZE; + previous_block = classified.first >> (SIZE - 1); + } + + Ok(None) + } +} + +impl<'i, 'b, 'r, I, R> Memmem<'i, 'b, 'r, I, SIZE> for Avx512MemmemClassifier64<'i, 'b, 'r, I, R> +where + I: Input, + R: InputRecorder>, + 'i: 'r, +{ + #[inline(always)] + fn find_label( + &mut self, + first_block: Option>, + start_idx: usize, + label: &JsonString, + ) -> Result)>, InputError> { + if let Some(b) = first_block { + if let Some(res) = shared::find_label_in_first_block(self.input, b, start_idx, label)? { + return Ok(Some(res)); + } + } + let next_block_offset = self.iter.get_offset(); + // SAFETY: target feature invariant + unsafe { self.find_label_avx512(label, next_block_offset) } + } +} diff --git a/crates/rsonpath-lib/src/classification/memmem/neon_32.rs b/crates/rsonpath-lib/src/classification/memmem/neon_32.rs new file mode 100644 index 00000000..2138ebf3 --- /dev/null +++ b/crates/rsonpath-lib/src/classification/memmem/neon_32.rs @@ -0,0 +1,183 @@ +use super::{shared::mask_32, shared::vector_neon, *}; +use crate::{ + classification::mask::m32, + input::{error::InputErrorConvertible, InputBlock, InputBlockIterator}, +}; + +const SIZE: usize = 32; + +pub(crate) struct Constructor; + +impl MemmemImpl for Constructor { + type Classifier<'i, 'b, 'r, I, R> = NeonMemmemClassifier<'i, 'b, 'r, I, R> + where + I: Input + 'i, + ::BlockIterator<'i, 'r, R, BLOCK_SIZE>: 'b, + R: InputRecorder<::Block<'i, BLOCK_SIZE>> + 'r, + 'i: 'r; + + fn memmem<'i, 'b, 'r, I, R>( + input: &'i I, + iter: &'b mut ::BlockIterator<'i, 'r, R, BLOCK_SIZE>, + ) -> Self::Classifier<'i, 'b, 'r, I, R> + where + I: Input, + R: InputRecorder<::Block<'i, BLOCK_SIZE>>, + 'i: 'r, + { + Self::Classifier { input, iter } + } +} + +pub(crate) struct NeonMemmemClassifier<'i, 'b, 'r, I, R> +where + I: Input, + R: InputRecorder> + 'r, +{ + input: &'i I, + iter: &'b mut I::BlockIterator<'i, 'r, R, SIZE>, +} + +impl<'i, 'b, 'r, I, R> NeonMemmemClassifier<'i, 'b, 'r, I, R> +where + I: Input, + R: InputRecorder>, + 'i: 'r, +{ + #[inline] + #[allow(dead_code)] + pub(crate) fn new(input: &'i I, iter: &'b mut I::BlockIterator<'i, 'r, R, SIZE>) -> Self { + Self { input, iter } + } + + #[inline(always)] + unsafe fn find_empty( + &mut self, + label: &JsonString, + mut offset: usize, + ) -> Result)>, InputError> { + let classifier = vector_neon::BlockClassifierNeon::new(b'"', b'"'); + let mut previous_block: u32 = 0; + + while let Some(block) = self.iter.next().e()? { + let (block1, block2) = block.halves(); + let classified1 = classifier.classify_block(block1); + let classified2 = classifier.classify_block(block2); + + let first_bitmask = m32::combine_16(classified1.first, classified2.first); + let second_bitmask = m32::combine_16(classified1.second, classified2.second); + + let mut result = (previous_block | (first_bitmask << 1)) & second_bitmask; + while result != 0 { + let idx = result.trailing_zeros() as usize; + if self + .input + .is_member_match(offset + idx - 1, offset + idx + 1, label) + .e()? + { + return Ok(Some((offset + idx - 1, block))); + } + result &= !(1 << idx); + } + + offset += SIZE; + previous_block = first_bitmask >> (SIZE - 1); + } + + Ok(None) + } + + // Here we want to detect the pattern `"c"` + // For interblock communication we need to bit of information that requires extra work to get obtained. + // one for the block cut being `"` and `c"` and one for `"c` and `"`. We only deal with one of them. + #[inline(always)] + unsafe fn find_letter( + &mut self, + label: &JsonString, + mut offset: usize, + ) -> Result)>, InputError> { + let classifier = vector_neon::BlockClassifierNeon::new(label.unquoted().as_bytes()[0], b'"'); + let mut previous_block: u32 = 0; + + while let Some(block) = self.iter.next().e()? { + let (block1, block2) = block.halves(); + let classified1 = classifier.classify_block(block1); + let classified2 = classifier.classify_block(block2); + + let first_bitmask = m32::combine_16(classified1.first, classified2.first); + let second_bitmask = m32::combine_16(classified1.second, classified2.second); + + if let Some(res) = + mask_32::find_in_mask(self.input, label, previous_block, first_bitmask, second_bitmask, offset)? + { + return Ok(Some((res, block))); + } + + offset += SIZE; + previous_block = first_bitmask >> (SIZE - 1); + } + + Ok(None) + } + + #[inline(always)] + unsafe fn find_label_neon( + &mut self, + label: &JsonString, + mut offset: usize, + ) -> Result)>, InputError> { + if label.unquoted().is_empty() { + return self.find_empty(label, offset); + } else if label.unquoted().len() == 1 { + return self.find_letter(label, offset); + } + + let classifier = + vector_neon::BlockClassifierNeon::new(label.unquoted().as_bytes()[0], label.unquoted().as_bytes()[1]); + let mut previous_block: u32 = 0; + + while let Some(block) = self.iter.next().e()? { + let (block1, block2) = block.halves(); + let classified1 = classifier.classify_block(block1); + let classified2 = classifier.classify_block(block2); + + let first_bitmask = m32::combine_16(classified1.first, classified2.first); + let second_bitmask = m32::combine_16(classified1.second, classified2.second); + + if let Some(res) = + mask_32::find_in_mask(self.input, label, previous_block, first_bitmask, second_bitmask, offset)? + { + return Ok(Some((res, block))); + } + + offset += SIZE; + previous_block = first_bitmask >> (SIZE - 1); + } + + Ok(None) + } +} + +impl<'i, 'b, 'r, I, R> Memmem<'i, 'b, 'r, I, SIZE> for NeonMemmemClassifier<'i, 'b, 'r, I, R> +where + I: Input, + R: InputRecorder>, + 'i: 'r, +{ + #[inline(always)] + fn find_label( + &mut self, + first_block: Option>, + start_idx: usize, + label: &JsonString, + ) -> Result)>, InputError> { + if let Some(b) = first_block { + if let Some(res) = shared::find_label_in_first_block(self.input, b, start_idx, label)? { + return Ok(Some(res)); + } + } + let next_block_offset = self.iter.get_offset(); + // SAFETY: target feature invariant + unsafe { self.find_label_neon(label, next_block_offset) } + } +} diff --git a/crates/rsonpath-lib/src/classification/memmem/neon_64.rs b/crates/rsonpath-lib/src/classification/memmem/neon_64.rs new file mode 100644 index 00000000..aadae4e4 --- /dev/null +++ b/crates/rsonpath-lib/src/classification/memmem/neon_64.rs @@ -0,0 +1,219 @@ +use super::{shared::mask_64, shared::vector_neon, *}; +use crate::{ + classification::mask::m64, + input::{error::InputErrorConvertible, InputBlock, InputBlockIterator}, +}; + +const SIZE: usize = 64; + +pub(crate) struct Constructor; + +impl MemmemImpl for Constructor { + type Classifier<'i, 'b, 'r, I, R> = NeonMemmemClassifier64<'i, 'b, 'r, I, R> + where + I: Input + 'i, + ::BlockIterator<'i, 'r, R, BLOCK_SIZE>: 'b, + R: InputRecorder<::Block<'i, BLOCK_SIZE>> + 'r, + 'i: 'r; + + fn memmem<'i, 'b, 'r, I, R>( + input: &'i I, + iter: &'b mut ::BlockIterator<'i, 'r, R, BLOCK_SIZE>, + ) -> Self::Classifier<'i, 'b, 'r, I, R> + where + I: Input, + R: InputRecorder<::Block<'i, BLOCK_SIZE>>, + 'i: 'r, + { + Self::Classifier { input, iter } + } +} + +pub(crate) struct NeonMemmemClassifier64<'i, 'b, 'r, I, R> +where + I: Input, + R: InputRecorder> + 'r, +{ + input: &'i I, + iter: &'b mut I::BlockIterator<'i, 'r, R, SIZE>, +} + +impl<'i, 'b, 'r, I, R> NeonMemmemClassifier64<'i, 'b, 'r, I, R> +where + I: Input, + R: InputRecorder>, + 'i: 'r, +{ + #[inline] + #[allow(dead_code)] + pub(crate) fn new(input: &'i I, iter: &'b mut I::BlockIterator<'i, 'r, R, SIZE>) -> Self { + Self { input, iter } + } + + #[inline(always)] + unsafe fn find_empty( + &mut self, + label: &JsonString, + mut offset: usize, + ) -> Result)>, InputError> { + let classifier = vector_neon::BlockClassifierNeon::new(b'"', b'"'); + let mut previous_block: u64 = 0; + + while let Some(block) = self.iter.next().e()? { + let (block1, block2, block3, block4) = block.quarters(); + let classified1 = classifier.classify_block(block1); + let classified2 = classifier.classify_block(block2); + let classified3 = classifier.classify_block(block3); + let classified4 = classifier.classify_block(block4); + + let first_bitmask = m64::combine_16( + classified1.first, + classified2.first, + classified3.first, + classified4.first, + ); + let second_bitmask = m64::combine_16( + classified1.second, + classified2.second, + classified3.second, + classified4.second, + ); + + let mut result = (previous_block | (first_bitmask << 1)) & second_bitmask; + while result != 0 { + let idx = result.trailing_zeros() as usize; + if self + .input + .is_member_match(offset + idx - 1, offset + idx + 1, label) + .e()? + { + return Ok(Some((offset + idx - 1, block))); + } + result &= !(1 << idx); + } + + offset += SIZE; + previous_block = first_bitmask >> (SIZE - 1); + } + + Ok(None) + } + + // Here we want to detect the pattern `"c"` + // For interblock communication we need to bit of information that requires extra work to get obtained. + // one for the block cut being `"` and `c"` and one for `"c` and `"`. We only deal with one of them. + #[inline(always)] + unsafe fn find_letter( + &mut self, + label: &JsonString, + mut offset: usize, + ) -> Result)>, InputError> { + let classifier = vector_neon::BlockClassifierNeon::new(label.unquoted().as_bytes()[0], b'"'); + let mut previous_block: u64 = 0; + + while let Some(block) = self.iter.next().e()? { + let (block1, block2, block3, block4) = block.quarters(); + let classified1 = classifier.classify_block(block1); + let classified2 = classifier.classify_block(block2); + let classified3 = classifier.classify_block(block3); + let classified4 = classifier.classify_block(block4); + + let first_bitmask = m64::combine_16( + classified1.first, + classified2.first, + classified3.first, + classified4.first, + ); + let second_bitmask = m64::combine_16( + classified1.second, + classified2.second, + classified3.second, + classified4.second, + ); + + if let Some(res) = + mask_64::find_in_mask(self.input, label, previous_block, first_bitmask, second_bitmask, offset)? + { + return Ok(Some((res, block))); + } + + offset += SIZE; + previous_block = first_bitmask >> (SIZE - 1); + } + + Ok(None) + } + + #[inline(always)] + unsafe fn find_label_neon( + &mut self, + label: &JsonString, + mut offset: usize, + ) -> Result)>, InputError> { + if label.unquoted().is_empty() { + return self.find_empty(label, offset); + } else if label.unquoted().len() == 1 { + return self.find_letter(label, offset); + } + + let classifier = + vector_neon::BlockClassifierNeon::new(label.unquoted().as_bytes()[0], label.unquoted().as_bytes()[1]); + let mut previous_block: u64 = 0; + + while let Some(block) = self.iter.next().e()? { + let (block1, block2, block3, block4) = block.quarters(); + let classified1 = classifier.classify_block(block1); + let classified2 = classifier.classify_block(block2); + let classified3 = classifier.classify_block(block3); + let classified4 = classifier.classify_block(block4); + + let first_bitmask = m64::combine_16( + classified1.first, + classified2.first, + classified3.first, + classified4.first, + ); + let second_bitmask = m64::combine_16( + classified1.second, + classified2.second, + classified3.second, + classified4.second, + ); + + if let Some(res) = + mask_64::find_in_mask(self.input, label, previous_block, first_bitmask, second_bitmask, offset)? + { + return Ok(Some((res, block))); + } + + offset += SIZE; + previous_block = first_bitmask >> (SIZE - 1); + } + + Ok(None) + } +} + +impl<'i, 'b, 'r, I, R> Memmem<'i, 'b, 'r, I, SIZE> for NeonMemmemClassifier64<'i, 'b, 'r, I, R> +where + I: Input, + R: InputRecorder>, + 'i: 'r, +{ + #[inline(always)] + fn find_label( + &mut self, + first_block: Option>, + start_idx: usize, + label: &JsonString, + ) -> Result)>, InputError> { + if let Some(b) = first_block { + if let Some(res) = shared::find_label_in_first_block(self.input, b, start_idx, label)? { + return Ok(Some(res)); + } + } + let next_block_offset = self.iter.get_offset(); + // SAFETY: target feature invariant + unsafe { self.find_label_neon(label, next_block_offset) } + } +} diff --git a/crates/rsonpath-lib/src/classification/memmem/shared.rs b/crates/rsonpath-lib/src/classification/memmem/shared.rs index c077c5bb..59fabffd 100644 --- a/crates/rsonpath-lib/src/classification/memmem/shared.rs +++ b/crates/rsonpath-lib/src/classification/memmem/shared.rs @@ -6,12 +6,16 @@ use rsonpath_syntax::str::JsonString; #[cfg(target_arch = "x86")] pub(super) mod mask_32; -#[cfg(target_arch = "x86_64")] +#[cfg(any(target_arch = "x86_64", target_arch = "aarch64"))] pub(super) mod mask_64; #[cfg(any(target_arch = "x86", target_arch = "x86_64"))] pub(super) mod vector_128; #[cfg(any(target_arch = "x86", target_arch = "x86_64"))] pub(super) mod vector_256; +#[cfg(any(target_arch = "x86", target_arch = "x86_64"))] +pub(super) mod vector_512; +#[cfg(any(target_arch = "aarch64"))] +pub(super) mod vector_neon; pub(crate) fn find_label_in_first_block<'i, 'r, I, const N: usize>( input: &I, diff --git a/crates/rsonpath-lib/src/classification/memmem/shared/vector_512.rs b/crates/rsonpath-lib/src/classification/memmem/shared/vector_512.rs new file mode 100644 index 00000000..ec658d1f --- /dev/null +++ b/crates/rsonpath-lib/src/classification/memmem/shared/vector_512.rs @@ -0,0 +1,34 @@ +#[cfg(target_arch = "x86")] +use ::core::arch::x86::*; +#[cfg(target_arch = "x86_64")] +use ::core::arch::x86_64::*; + +pub(crate) struct BlockClassifier512 { + first: __m512i, + second: __m512i, +} + +impl BlockClassifier512 { + #[target_feature(enable = "avx512f")] + pub(crate) unsafe fn new(first: u8, second: u8) -> Self { + Self { + first: _mm512_set1_epi8(first as i8), + second: _mm512_set1_epi8(second as i8), + } + } + + #[target_feature(enable = "avx512f")] + pub(crate) unsafe fn classify_block(&self, block: &[u8]) -> BlockClassification512 { + let byte_vector = _mm512_loadu_si512(block.as_ptr().cast::()); + + let first = _mm512_cmpeq_epi8_mask(byte_vector, self.first); + let second = _mm512_cmpeq_epi8_mask(byte_vector, self.second); + + BlockClassification512 { first, second } + } +} + +pub(crate) struct BlockClassification512 { + pub(crate) first: u64, + pub(crate) second: u64, +} diff --git a/crates/rsonpath-lib/src/classification/memmem/shared/vector_neon.rs b/crates/rsonpath-lib/src/classification/memmem/shared/vector_neon.rs new file mode 100644 index 00000000..8d22914c --- /dev/null +++ b/crates/rsonpath-lib/src/classification/memmem/shared/vector_neon.rs @@ -0,0 +1,56 @@ +#[cfg(target_arch = "aarch64")] +use ::core::arch::aarch64::*; + +#[cfg(target_arch = "aarch64")] +use ::core::arch::aarch64::*; + +#[inline] +#[target_feature(enable = "neon")] +unsafe fn neon_movemask_epi8(cmp_vector: uint8x16_t) -> i16 { + let shift_values: [i8; 16] = [-7, -6, -5, -4, -3, -2, -1, 0, -7, -6, -5, -4, -3, -2, -1, 0]; + + let vshift = vld1q_s8(shift_values.as_ptr()); + + let vmask = vandq_u8(cmp_vector, vdupq_n_u8(0x80)); + + let vmask = vshlq_u8(vmask, vshift); + + let low = vaddv_u8(vget_low_u8(vmask)) as i16; + let high = vaddv_u8(vget_high_u8(vmask)) as i16; + + low | (high << 8) +} + +pub(crate) struct BlockClassifierNeon { + first: int8x16_t, + second: int8x16_t, +} + +impl BlockClassifierNeon { + #[target_feature(enable = "neon")] + pub(crate) unsafe fn new(first: u8, second: u8) -> Self { + Self { + first: vdupq_n_s8(first as i8), + second: vdupq_n_s8(second as i8), + } + } + + #[target_feature(enable = "neon")] + pub(crate) unsafe fn classify_block(&self, block: &[u8]) -> BlockClassificationNeon { + // vld1q zakłada alignment, ale na nowszych powinno działać + let byte_vector = vld1q_s8(block.as_ptr().cast::()); + + let first_cmp_vector = vceqq_s8(byte_vector, self.first); + let second_cmp_vector = vceqq_s8(byte_vector, self.second); + + let first = neon_movemask_epi8(first_cmp_vector) as u16; + let second = neon_movemask_epi8(second_cmp_vector) as u16; + + BlockClassificationNeon { first, second } + } +} + +pub(crate) struct BlockClassificationNeon { + pub(crate) first: u16, + pub(crate) second: u16, +} diff --git a/crates/rsonpath-lib/src/classification/quotes.rs b/crates/rsonpath-lib/src/classification/quotes.rs index 646c36c2..8cc0953e 100644 --- a/crates/rsonpath-lib/src/classification/quotes.rs +++ b/crates/rsonpath-lib/src/classification/quotes.rs @@ -97,6 +97,8 @@ where pub(crate) mod nosimd; pub(crate) mod shared; +#[cfg(target_arch = "x86_64")] +pub(crate) mod avx512_64; #[cfg(target_arch = "x86")] pub(crate) mod avx2_32; #[cfg(target_arch = "x86_64")] @@ -105,6 +107,8 @@ pub(crate) mod avx2_64; pub(crate) mod sse2_32; #[cfg(target_arch = "x86_64")] pub(crate) mod sse2_64; +#[cfg(target_arch = "aarch64")] +pub(crate) mod neon_64; pub(crate) trait QuotesImpl { type Classifier<'i, I>: QuoteClassifiedIterator<'i, I, MaskType, BLOCK_SIZE> + InnerIter diff --git a/crates/rsonpath-lib/src/classification/quotes/avx512_64.rs b/crates/rsonpath-lib/src/classification/quotes/avx512_64.rs new file mode 100644 index 00000000..b64c7027 --- /dev/null +++ b/crates/rsonpath-lib/src/classification/quotes/avx512_64.rs @@ -0,0 +1,59 @@ +use super::{ + shared::{mask_64, vector_512}, + *, +}; +use crate::{block, classification::mask::m64, debug, input::error::InputErrorConvertible}; +use std::marker::PhantomData; + +super::shared::quote_classifier!(Avx512QuoteClassifier64, BlockAvx512Classifier, 64, u64); + +struct BlockAvx512Classifier { + internal_classifier: mask_64::BlockClassifier64Bit, +} + +impl BlockAvx512Classifier { + fn new() -> Self { + Self { + internal_classifier: mask_64::BlockClassifier64Bit::new(), + } + } + + #[inline(always)] + unsafe fn classify<'a, B: InputBlock<'a, 64>>(&mut self, block: &B) -> u64 { + block!(block[..64]); + + let classification = vector_512::classify_block(block); + + let slashes = classification.slashes; + let quotes = classification.quotes; + + self.internal_classifier.classify(slashes, quotes) + } +} + +// FIXME add test with 512 bytes. +#[cfg(all(test, cfg = "avx_512"))] +mod tests { + use super::Avx512QuoteClassifier64; + use crate::{ + input::{Input, OwnedBytes}, + result::empty::EmptyRecorder, + FallibleIterator, + }; + use test_case::test_case; + + #[test_case("" => None)] + #[test_case("abcd" => Some(0))] + #[test_case(r#""abcd""# => Some(0b01_1111))] + #[test_case(r#""number": 42, "string": "something" "# => Some(0b0011_1111_1111_0001_1111_1100_0000_0111_1111))] + #[test_case(r#"abc\"abc\""# => Some(0b00_0000_0000))] + #[test_case(r#"abc\\"abc\\""# => Some(0b0111_1110_0000))] + #[test_case(r#"{"aaa":[{},{"b":{"c":[1,2,3]}}],"e":{"a":[[],[1,2,3],"# => Some(0b0_0000_0000_0000_0110_0011_0000_0000_0000_0110_0011_0000_0001_1110))] + fn single_block(str: &str) -> Option { + let owned_str = str.to_owned(); + let input = OwnedBytes::new(&owned_str).unwrap(); + let iter = input.iter_blocks::<_, 64>(&EmptyRecorder); + let mut classifier = Avx512QuoteClassifier64::new(iter); + classifier.next().unwrap().map(|x| x.within_quotes_mask) + } +} diff --git a/crates/rsonpath-lib/src/classification/quotes/neon_64.rs b/crates/rsonpath-lib/src/classification/quotes/neon_64.rs new file mode 100644 index 00000000..897c0ae4 --- /dev/null +++ b/crates/rsonpath-lib/src/classification/quotes/neon_64.rs @@ -0,0 +1,73 @@ +use super::{ + shared::{mask_neon, vector_neon}, + *, +}; +use crate::{block, classification::mask::m64, debug, input::error::InputErrorConvertible}; +use std::marker::PhantomData; + +super::shared::quote_classifier!(NeonQuoteClassifier, BlockNeonClassifier, 64, u64); + +struct BlockNeonClassifier { + internal_classifier: mask_neon::BlockClassifierNeon, +} + +impl BlockNeonClassifier { + fn new() -> Self { + Self { + internal_classifier: mask_neon::BlockClassifierNeon::new(), + } + } + + #[inline(always)] + unsafe fn classify<'a, B: InputBlock<'a, 64>>(&mut self, blocks: &B) -> u64 { + block!(blocks[..64]); + + let (block1, block2, block3, block4) = blocks.quarters(); + + let classification1 = vector_neon::classify_block(block1); + let classification2 = vector_neon::classify_block(block2); + let classification3 = vector_neon::classify_block(block3); + let classification4 = vector_neon::classify_block(block4); + + let slashes = m64::combine_16( + classification1.slashes, + classification2.slashes, + classification3.slashes, + classification4.slashes, + ); + let quotes = m64::combine_16( + classification1.quotes, + classification2.quotes, + classification3.quotes, + classification4.quotes, + ); + + self.internal_classifier.classify(slashes, quotes) + } +} + +#[cfg(all(test, cfg = "neon"))] +mod tests { + use super::NeonQuoteClassifier; + use crate::{ + input::{Input, OwnedBytes}, + result::empty::EmptyRecorder, + FallibleIterator, + }; + use test_case::test_case; + + #[test_case("" => None)] + #[test_case("abcd" => Some(0))] + #[test_case(r#""abcd""# => Some(0b01_1111))] + #[test_case(r#""number": 42, "string": "something" "# => Some(0b0011_1111_1111_0001_1111_1100_0000_0111_1111))] + #[test_case(r#"abc\"abc\""# => Some(0b00_0000_0000))] + #[test_case(r#"abc\\"abc\\""# => Some(0b0111_1110_0000))] + #[test_case(r#"{"aaa":[{},{"b":{"c":[1,2,3]}}],"e":{"a":[[],[1,2,3],"# => Some(0b0_0000_0000_0000_0110_0011_0000_0000_0000_0110_0011_0000_0001_1110))] + fn single_block(str: &str) -> Option { + let owned_str = str.to_owned(); + let input = OwnedBytes::new(&owned_str).unwrap(); + let iter = input.iter_blocks::<_, 64>(&EmptyRecorder); + let mut classifier = NeonQuoteClassifier::new(iter); + classifier.next().unwrap().map(|x| x.within_quotes_mask) + } +} diff --git a/crates/rsonpath-lib/src/classification/quotes/shared.rs b/crates/rsonpath-lib/src/classification/quotes/shared.rs index 2f6d15bb..586987b6 100644 --- a/crates/rsonpath-lib/src/classification/quotes/shared.rs +++ b/crates/rsonpath-lib/src/classification/quotes/shared.rs @@ -2,10 +2,16 @@ pub(super) mod mask_32; #[cfg(target_arch = "x86_64")] pub(super) mod mask_64; +#[cfg(target_arch = "aarch64")] +pub(super) mod mask_neon; #[cfg(any(target_arch = "x86", target_arch = "x86_64"))] pub(super) mod vector_128; #[cfg(any(target_arch = "x86", target_arch = "x86_64"))] pub(super) mod vector_256; +#[cfg(any(target_arch = "x86", target_arch = "x86_64"))] +pub(super) mod vector_512; +#[cfg(target_arch = "aarch64")] +pub(super) mod vector_neon; #[allow(unused_macros)] macro_rules! quote_classifier { diff --git a/crates/rsonpath-lib/src/classification/quotes/shared/mask_neon.rs b/crates/rsonpath-lib/src/classification/quotes/shared/mask_neon.rs new file mode 100644 index 00000000..53ffbcb0 --- /dev/null +++ b/crates/rsonpath-lib/src/classification/quotes/shared/mask_neon.rs @@ -0,0 +1,90 @@ +use crate::bin_u64; +#[cfg(target_arch = "aarch64")] +use ::core::arch::aarch64::*; + +/// Bitmask selecting bits on even positions when indexing from zero. +pub(crate) const ODD: u64 = 0b0101_0101_0101_0101_0101_0101_0101_0101_0101_0101_0101_0101_0101_0101_0101_0101_u64; +/// Bitmask selecting bits on odd positions when indexing from zero. +pub(crate) const EVEN: u64 = 0b1010_1010_1010_1010_1010_1010_1010_1010_1010_1010_1010_1010_1010_1010_1010_1010_u64; + +#[target_feature(enable = "neon")] +unsafe fn all_ones128() -> u64 { + !0 +} + +pub(crate) struct BlockClassifierNeon { + /// Compressed information about the state from the previous block. + /// The first bit is lit iff the previous block ended with an unescaped escape character. + /// The second bit is lit iff the previous block ended with a starting quote, + /// meaning that it was not escaped, nor was it the closing quote of a quoted sequence. + prev_block_mask: u8, +} + +impl BlockClassifierNeon { + pub(crate) fn new() -> Self { + Self { prev_block_mask: 0 } + } + + /// Set the inter-block state based on slash overflow and the quotes mask. + fn update_prev_block_mask(&mut self, set_slash_mask: bool, quotes: u64) { + let slash_mask = u8::from(set_slash_mask); + let quote_mask = (((quotes & (1 << 63)) >> 62) as u8) & 0x02; + self.prev_block_mask = slash_mask | quote_mask; + } + + /// Flip the inter-block state bit representing the quote state. + pub(crate) fn flip_prev_quote_mask(&mut self) { + self.prev_block_mask ^= 0x02; + } + + /// Returns 0x01 if the last character of the previous block was an unescaped escape character, + /// zero otherwise. + fn get_prev_slash_mask(&self) -> u64 { + u64::from(self.prev_block_mask & 0x01) + } + + /// Returns 0x01 if the last character of the previous block was an unescaped quote, zero otherwise. + fn get_prev_quote_mask(&self) -> u64 { + u64::from((self.prev_block_mask & 0x02) >> 1) + } + + #[target_feature(enable = "neon,aes")] + pub(crate) unsafe fn classify(&mut self, slashes: u64, quotes: u64) -> u64 { + let (escaped, set_prev_slash_mask) = if slashes == 0 { + (self.get_prev_slash_mask(), false) + } else { + let slashes_excluding_escaped_first = slashes & !self.get_prev_slash_mask(); + let starts = slashes_excluding_escaped_first & !(slashes_excluding_escaped_first << 1); + let odd_starts = ODD & starts; + let even_starts = EVEN & starts; + + let odd_starts_carry = odd_starts.wrapping_add(slashes); + let (even_starts_carry, set_prev_slash_mask) = even_starts.overflowing_add(slashes); + + let ends_of_odd_starts = odd_starts_carry & !slashes; + let ends_of_even_starts = even_starts_carry & !slashes; + + let escaped = (ends_of_odd_starts & EVEN) | (ends_of_even_starts & ODD) | self.get_prev_slash_mask(); + + (escaped, set_prev_slash_mask) + }; + + let nonescaped_quotes = (quotes & !escaped) ^ self.get_prev_quote_mask(); + + let cumulative_xor = vmull_p64(nonescaped_quotes as u64, all_ones128()); + + let within_quotes = cumulative_xor as u64; + self.update_prev_block_mask(set_prev_slash_mask, within_quotes); + + bin_u64!("slashes", slashes); + bin_u64!("quotes", quotes); + bin_u64!("prev_slash_bit", self.get_prev_slash_mask()); + bin_u64!("prev_quote_bit", self.get_prev_quote_mask()); + bin_u64!("escaped", escaped); + bin_u64!("quotes & !escaped", quotes & !escaped); + bin_u64!("nonescaped_quotes", nonescaped_quotes); + bin_u64!("within_quotes", within_quotes); + + within_quotes + } +} diff --git a/crates/rsonpath-lib/src/classification/quotes/shared/vector_512.rs b/crates/rsonpath-lib/src/classification/quotes/shared/vector_512.rs new file mode 100644 index 00000000..b88935d4 --- /dev/null +++ b/crates/rsonpath-lib/src/classification/quotes/shared/vector_512.rs @@ -0,0 +1,31 @@ +#[cfg(target_arch = "x86")] +use ::core::arch::x86::*; +#[cfg(target_arch = "x86_64")] +use ::core::arch::x86_64::*; + +#[inline(always)] +pub(crate) unsafe fn quote_mask() -> __m512i { + _mm512_set1_epi8(b'"' as i8) +} + +#[inline(always)] +pub(crate) unsafe fn slash_mask() -> __m512i { + _mm512_set1_epi8(b'\\' as i8) +} + +#[target_feature(enable = "avx512f")] +#[target_feature(enable = "avx512bw")] +pub(crate) unsafe fn classify_block(block: &[u8]) -> BlockClassification512 { + let byte_vector = _mm512_loadu_si512(block.as_ptr().cast::()); + + let slashes = _mm512_cmpeq_epi8_mask(byte_vector, slash_mask()); + + let quotes = _mm512_cmpeq_epi8_mask(byte_vector, quote_mask()); + + BlockClassification512 { slashes, quotes } +} + +pub(crate) struct BlockClassification512 { + pub(crate) slashes: u64, + pub(crate) quotes: u64, +} diff --git a/crates/rsonpath-lib/src/classification/quotes/shared/vector_neon.rs b/crates/rsonpath-lib/src/classification/quotes/shared/vector_neon.rs new file mode 100644 index 00000000..30dd4aac --- /dev/null +++ b/crates/rsonpath-lib/src/classification/quotes/shared/vector_neon.rs @@ -0,0 +1,52 @@ +#[cfg(target_arch = "aarch64")] +use ::core::arch::aarch64::*; + +#[inline] +#[target_feature(enable = "neon")] +unsafe fn neon_movemask_epi8(cmp_vector: uint8x16_t) -> i16 { + // Tablica shiftów + let shift_values: [i8; 16] = [-7, -6, -5, -4, -3, -2, -1, 0, -7, -6, -5, -4, -3, -2, -1, 0]; + + // Załaduj shift do NEON + let vshift = vld1q_s8(shift_values.as_ptr()); + + // Zamaskowanie bitu 7 (0x80) + let vmask = vandq_u8(cmp_vector, vdupq_n_u8(0x80)); + + // Przesunięcie bitów + let vmask = vshlq_u8(vmask, vshift); + + // Sumowanie dolnej i górnej połowy + let low = vaddv_u8(vget_low_u8(vmask)) as i16; + let high = vaddv_u8(vget_high_u8(vmask)) as i16; + + low | (high << 8) +} + +#[inline(always)] +pub(crate) unsafe fn quote_mask() -> int8x16_t { + vdupq_n_s8(b'"' as i8) +} + +#[inline(always)] +pub(crate) unsafe fn slash_mask() -> int8x16_t { + vdupq_n_s8(b'\\' as i8) +} + +#[target_feature(enable = "neon")] +pub(crate) unsafe fn classify_block(block: &[u8]) -> BlockClassificationNeon { + let byte_vector = vreinterpretq_s8_u8(vld1q_u8(block.as_ptr())); + + let slash_cmp = vceqq_s8(byte_vector, slash_mask()); + let slashes = neon_movemask_epi8(slash_cmp) as u16; + + let quote_cmp = vceqq_s8(byte_vector, quote_mask()); + let quotes = neon_movemask_epi8(quote_cmp) as u16; + + BlockClassificationNeon { slashes, quotes } +} + +pub(crate) struct BlockClassificationNeon { + pub(crate) slashes: u16, + pub(crate) quotes: u16, +} diff --git a/crates/rsonpath-lib/src/classification/simd.rs b/crates/rsonpath-lib/src/classification/simd.rs index a869ac34..4214de60 100644 --- a/crates/rsonpath-lib/src/classification/simd.rs +++ b/crates/rsonpath-lib/src/classification/simd.rs @@ -454,6 +454,10 @@ pub(crate) enum SimdTag { Ssse3, /// AVX2 detected. Avx2, + /// AVX512 detected. + Avx512, + /// ARM NEON detected. + Neon, } /// Runtime-detected SIMD configuration guiding how to construct a [`Simd`] implementation for the engine. @@ -496,6 +500,8 @@ impl SimdConfiguration { "sse2" => Some(SimdTag::Sse2), "ssse3" => Some(SimdTag::Ssse3), "avx2" => Some(SimdTag::Avx2), + "avx512" => Some(SimdTag::Avx512), + "neon" => Some(SimdTag::Neon), _ => None, }; let quotes = match quotes_str.to_ascii_lowercase().as_ref() { @@ -537,7 +543,14 @@ pub(crate) fn configure() -> SimdConfiguration { } cfg_if! { - if #[cfg(not(feature = "simd"))] + // AArch64? + if #[cfg(any(target_arch = "aarch64"))] + { + let highest_simd = SimdTag::Neon; + let fast_quotes = false; + let fast_popcnt = false; + } + else if #[cfg(not(feature = "simd"))] { let highest_simd = SimdTag::Nosimd; let fast_quotes = false; @@ -545,7 +558,9 @@ pub(crate) fn configure() -> SimdConfiguration { } else if #[cfg(any(target_arch = "x86", target_arch = "x86_64"))] { - let highest_simd = if is_x86_feature_detected!("avx2") { + let highest_simd = if is_x86_feature_detected!("avx512f") && is_x86_feature_detected!("avx512bw") { + SimdTag::Avx512 + } else if is_x86_feature_detected!("avx2") { SimdTag::Avx2 } else if is_x86_feature_detected!("ssse3") { SimdTag::Ssse3 @@ -581,6 +596,8 @@ impl Display for SimdConfiguration { SimdTag::Sse2 => "sse2", SimdTag::Ssse3 => "ssse3", SimdTag::Avx2 => "avx2", + SimdTag::Avx512 => "avx512", + SimdTag::Neon => "neon", }; let quote_desc = if self.fast_quotes { "fast_quotes" } else { "slow_quotes" }; let popcnt_desc = if self.fast_popcnt { "fast_popcnt" } else { "slow_popcnt" }; @@ -592,19 +609,46 @@ impl Display for SimdConfiguration { pub(crate) const NOSIMD: usize = 0; cfg_if! { - if #[cfg(any(target_arch = "x86_64", target_arch = "x86"))] { - pub(crate) const AVX2_PCLMULQDQ_POPCNT: usize = 1; - pub(crate) const SSSE3_PCLMULQDQ_POPCNT: usize = 2; - pub(crate) const SSSE3_PCLMULQDQ: usize = 3; - pub(crate) const SSSE3_POPCNT: usize = 4; - pub(crate) const SSSE3: usize = 5; - pub(crate) const SSE2_PCLMULQDQ_POPCNT: usize = 6; - pub(crate) const SSE2_PCLMULQDQ: usize = 7; - pub(crate) const SSE2_POPCNT: usize = 8; - pub(crate) const SSE2: usize = 9; + if #[cfg(any(target_arch = "aarch64"))] { + pub(crate) const NEON: usize = 11; macro_rules! dispatch_simd { ($simd:expr; $( $arg:expr ),* => fn $( $fn:tt )*) => {{ + #[target_feature(enable = "neon")] + #[target_feature(enable = "neon,aes")] + #[target_feature(enable = "neon")] + unsafe fn neon $($fn)* + + let simd = $simd; + + // SAFETY: depends on the provided SimdConfig, which cannot be incorrectly constructed. + unsafe { + match simd.dispatch_tag() { + _ => neon($($arg),*), + } + } + }}; + } + } + else if #[cfg(any(target_arch = "x86_64", target_arch = "x86"))] { + pub(crate) const AVX512_PCLMULQDQ_POPCNT: usize = 1; + pub(crate) const AVX2_PCLMULQDQ_POPCNT: usize = 2; + pub(crate) const SSSE3_PCLMULQDQ_POPCNT: usize = 3; + pub(crate) const SSSE3_PCLMULQDQ: usize = 4; + pub(crate) const SSSE3_POPCNT: usize = 5; + pub(crate) const SSSE3: usize = 6; + pub(crate) const SSE2_PCLMULQDQ_POPCNT: usize = 7; + pub(crate) const SSE2_PCLMULQDQ: usize = 8; + pub(crate) const SSE2_POPCNT: usize = 9; + pub(crate) const SSE2: usize = 10; + + macro_rules! dispatch_simd { + ($simd:expr; $( $arg:expr ),* => fn $( $fn:tt )*) => {{ + #[target_feature(enable = "avx512f")] + #[target_feature(enable = "avx512bw")] + #[target_feature(enable = "pclmulqdq")] + #[target_feature(enable = "popcnt")] + unsafe fn avx512_pclmulqdq_popcnt $($fn)* #[target_feature(enable = "avx2")] #[target_feature(enable = "pclmulqdq")] #[target_feature(enable = "popcnt")] @@ -640,6 +684,7 @@ cfg_if! { // SAFETY: depends on the provided SimdConfig, which cannot be incorrectly constructed. unsafe { match simd.dispatch_tag() { + $crate::classification::simd::AVX512_PCLMULQDQ_POPCNT => avx512_pclmulqdq_popcnt($($arg),*), $crate::classification::simd::AVX2_PCLMULQDQ_POPCNT => avx2_pclmulqdq_popcnt($($arg),*), $crate::classification::simd::SSSE3_PCLMULQDQ_POPCNT => ssse3_pclmulqdq_popcnt($($arg),*), $crate::classification::simd::SSSE3_PCLMULQDQ => ssse3_pclmulqdq($($arg),*), @@ -673,7 +718,20 @@ cfg_if! { let conf = $conf; match conf.highest_simd() { - // AVX2 implies all other optimizations. + // AVX512 implies all other optimizations. + $crate::classification::simd::SimdTag::Avx512 => { + assert!(conf.fast_quotes()); + assert!(conf.fast_popcnt()); + let $simd = $crate::classification::simd::ResolvedSimd::< + $crate::classification::quotes::avx512_64::Constructor, + $crate::classification::structural::avx512_64::Constructor, + $crate::classification::depth::avx512_64::Constructor, + $crate::classification::memmem::avx512_64::Constructor, + {$crate::classification::simd::AVX512_PCLMULQDQ_POPCNT}, + >::new(); + $b + } + // AVX2 implies all optimizations other than 512. $crate::classification::simd::SimdTag::Avx2 => { assert!(conf.fast_quotes()); assert!(conf.fast_popcnt()); @@ -778,7 +836,7 @@ cfg_if! { } } // nosimd denies all optimizations. - $crate::classification::simd::SimdTag::Nosimd => { + $crate::classification::simd::SimdTag::Nosimd | $crate::classification::simd::SimdTag::Neon => { let $simd = $crate::classification::simd::ResolvedSimd::< $crate::classification::quotes::nosimd::Constructor, $crate::classification::structural::nosimd::Constructor, @@ -801,7 +859,9 @@ cfg_if! { match conf.highest_simd() { // AVX2 implies all other optimizations. - $crate::classification::simd::SimdTag::Avx2 => { + // AVX512 on x86 is yet to be implemented. + $crate::classification::simd::SimdTag::Avx2 | + $crate::classification::simd::SimdTag::Avx512 => { assert!(conf.fast_quotes()); assert!(conf.fast_popcnt()); let $simd = $crate::classification::simd::ResolvedSimd::< @@ -920,6 +980,44 @@ cfg_if! { }; } } + else if #[cfg(target_arch = "aarch64")] { + macro_rules! config_simd { + ($conf:expr => |$simd:ident| $b:block) => { + { + let conf = $conf; + + match conf.highest_simd() { + // Neon TODO comment + $crate::classification::simd::SimdTag::Neon => { + assert!(!conf.fast_quotes()); + assert!(!conf.fast_popcnt()); + let $simd = $crate::classification::simd::ResolvedSimd::< + $crate::classification::quotes::neon_64::Constructor, + $crate::classification::structural::neon_64::Constructor, + $crate::classification::depth::neon_64::Constructor, + $crate::classification::memmem::neon_64::Constructor, + {$crate::classification::simd::NEON}, + >::new(); + $b + } + // nosimd denies all optimizations. + // detection of other options makes no sense in aarch64. +// $crate::classification::simd::SimdTag::Nosimd => { + _ => { + let $simd = $crate::classification::simd::ResolvedSimd::< + $crate::classification::quotes::nosimd::Constructor, + $crate::classification::structural::nosimd::Constructor, + $crate::classification::depth::nosimd::Constructor, + $crate::classification::memmem::nosimd::Constructor, + {$crate::classification::simd::NOSIMD} + >::new(); + $b + } + } + } + }; + } + } else { macro_rules! config_simd { ($conf:expr => |$simd:ident| $b:block) => { diff --git a/crates/rsonpath-lib/src/classification/structural.rs b/crates/rsonpath-lib/src/classification/structural.rs index 804fe1ce..bc6f02ae 100644 --- a/crates/rsonpath-lib/src/classification/structural.rs +++ b/crates/rsonpath-lib/src/classification/structural.rs @@ -172,6 +172,8 @@ where pub(crate) mod nosimd; pub(crate) mod shared; +#[cfg(target_arch = "x86_64")] +pub(crate) mod avx512_64; #[cfg(target_arch = "x86")] pub(crate) mod avx2_32; #[cfg(target_arch = "x86_64")] @@ -180,6 +182,8 @@ pub(crate) mod avx2_64; pub(crate) mod ssse3_32; #[cfg(target_arch = "x86_64")] pub(crate) mod ssse3_64; +#[cfg(target_arch = "aarch64")] +pub(crate) mod neon_64; pub(crate) trait StructuralImpl { type Classifier<'i, I, Q>: StructuralIterator<'i, I, Q, MaskType, BLOCK_SIZE> diff --git a/crates/rsonpath-lib/src/classification/structural/avx512_64.rs b/crates/rsonpath-lib/src/classification/structural/avx512_64.rs new file mode 100644 index 00000000..5b6b5a5c --- /dev/null +++ b/crates/rsonpath-lib/src/classification/structural/avx512_64.rs @@ -0,0 +1,42 @@ +use super::{ + shared::{mask_64, vector_512}, + *, +}; +use crate::{ + bin_u64, + classification::{QuoteClassifiedBlock, ResumeClassifierBlockState}, + debug, + input::InputBlock, +}; + +super::shared::structural_classifier!(Avx512Classifier64, BlockAvx512Classifier64, mask_64, 64, u64); + +struct BlockAvx512Classifier64 { + internal_classifier: vector_512::BlockClassifier512, +} + +impl BlockAvx512Classifier64 { + fn new() -> Self { + Self { + // SAFETY: target feature invariant + internal_classifier: unsafe { vector_512::BlockClassifier512::new() }, + } + } + + #[inline(always)] + unsafe fn classify<'i, B: InputBlock<'i, 64>>( + &mut self, + quote_classified_block: QuoteClassifiedBlock, + ) -> mask_64::StructuralsBlock { + let block = "e_classified_block.block; + let classification = self.internal_classifier.classify_block(block); + + let structural = classification.structural; + let nonquoted_structural = structural & !quote_classified_block.within_quotes_mask; + + bin_u64!("structural", structural); + bin_u64!("nonquoted_structural", nonquoted_structural); + + mask_64::StructuralsBlock::new(quote_classified_block, nonquoted_structural) + } +} diff --git a/crates/rsonpath-lib/src/classification/structural/neon_64.rs b/crates/rsonpath-lib/src/classification/structural/neon_64.rs new file mode 100644 index 00000000..52a8dea9 --- /dev/null +++ b/crates/rsonpath-lib/src/classification/structural/neon_64.rs @@ -0,0 +1,51 @@ +use super::{ + shared::{mask_64, vector_neon}, + *, +}; +use crate::{ + bin_u64, + classification::mask::m64, + classification::{QuoteClassifiedBlock, ResumeClassifierBlockState}, + debug, + input::InputBlock, +}; + +super::shared::structural_classifier!(NeonClassifier, BlockNeonClassifier, mask_64, 64, u64); + +struct BlockNeonClassifier { + internal_classifier: vector_neon::BlockClassifierNeon, +} + +impl BlockNeonClassifier { + fn new() -> Self { + Self { + // SAFETY: target feature invariant + internal_classifier: unsafe { vector_neon::BlockClassifierNeon::new() }, + } + } + + #[inline(always)] + unsafe fn classify<'i, B: InputBlock<'i, 64>>( + &mut self, + quote_classified_block: QuoteClassifiedBlock, + ) -> mask_64::StructuralsBlock { + let (block1, block2, block3, block4) = quote_classified_block.block.quarters(); + let classification1 = self.internal_classifier.classify_block(block1); + let classification2 = self.internal_classifier.classify_block(block2); + let classification3 = self.internal_classifier.classify_block(block3); + let classification4 = self.internal_classifier.classify_block(block4); + + let structural = m64::combine_16( + classification1.structural, + classification2.structural, + classification3.structural, + classification4.structural, + ); + let nonquoted_structural = structural & !quote_classified_block.within_quotes_mask; + + bin_u64!("structural", structural); + bin_u64!("nonquoted_structural", nonquoted_structural); + + mask_64::StructuralsBlock::new(quote_classified_block, nonquoted_structural) + } +} diff --git a/crates/rsonpath-lib/src/classification/structural/shared.rs b/crates/rsonpath-lib/src/classification/structural/shared.rs index 459489b8..96f04d85 100644 --- a/crates/rsonpath-lib/src/classification/structural/shared.rs +++ b/crates/rsonpath-lib/src/classification/structural/shared.rs @@ -1,11 +1,16 @@ #[cfg(target_arch = "x86")] pub(super) mod mask_32; -#[cfg(target_arch = "x86_64")] +#[cfg(any(target_arch = "x86_64", target_arch = "aarch64"))] pub(super) mod mask_64; #[cfg(any(target_arch = "x86", target_arch = "x86_64"))] pub(super) mod vector_128; #[cfg(any(target_arch = "x86", target_arch = "x86_64"))] pub(super) mod vector_256; +#[cfg(any(target_arch = "x86", target_arch = "x86_64"))] +pub(super) mod vector_512; +#[cfg(target_arch = "aarch64")] +pub(super) mod vector_neon; + #[allow(unused_macros)] macro_rules! structural_classifier { diff --git a/crates/rsonpath-lib/src/classification/structural/shared/vector_512.rs b/crates/rsonpath-lib/src/classification/structural/shared/vector_512.rs new file mode 100644 index 00000000..f51b931a --- /dev/null +++ b/crates/rsonpath-lib/src/classification/structural/shared/vector_512.rs @@ -0,0 +1,129 @@ +#[cfg(target_arch = "x86_64")] +use ::core::arch::x86_64::*; + +const LOWER_NIBBLE_MASK_ARRAY: [u8; 64] = [ + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0x03, 0x01, 0x02, 0x01, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0x03, 0x01, 0x02, 0x01, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0x03, 0x01, 0x02, 0x01, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0x03, 0x01, 0x02, 0x01, 0xff, 0xff, +]; +const UPPER_NIBBLE_MASK_ARRAY: [u8; 64] = [ + 0xfe, 0xfe, 0x10, 0x10, 0xfe, 0x01, 0xfe, 0x01, + 0xfe, 0xfe, 0xfe, 0xfe, 0xfe, 0xfe, 0xfe, 0xfe, + 0xfe, 0xfe, 0x10, 0x10, 0xfe, 0x01, 0xfe, 0x01, + 0xfe, 0xfe, 0xfe, 0xfe, 0xfe, 0xfe, 0xfe, 0xfe, + 0xfe, 0xfe, 0x10, 0x10, 0xfe, 0x01, 0xfe, 0x01, + 0xfe, 0xfe, 0xfe, 0xfe, 0xfe, 0xfe, 0xfe, 0xfe, + 0xfe, 0xfe, 0x10, 0x10, 0xfe, 0x01, 0xfe, 0x01, + 0xfe, 0xfe, 0xfe, 0xfe, 0xfe, 0xfe, 0xfe, 0xfe, +]; +const COMMAS_TOGGLE_MASK_ARRAY: [u8; 64] = [ + 0x00, 0x00, 0x12, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x12, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x12, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x12, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, +]; +const COLON_TOGGLE_MASK_ARRAY: [u8; 64] = [ + 0x00, 0x00, 0x00, 0x13, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x13, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x13, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x13, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, +]; + +#[target_feature(enable = "avx512f")] +#[inline] +pub(crate) unsafe fn upper_nibble_zeroing_mask() -> __m512i { + _mm512_set1_epi8(0x0F) +} + +#[target_feature(enable = "avx512f")] +#[inline] +pub(crate) unsafe fn lower_nibble_mask() -> __m512i { + _mm512_loadu_si512(LOWER_NIBBLE_MASK_ARRAY.as_ptr().cast::()) +} + +#[target_feature(enable = "avx512f")] +#[inline] +pub(crate) unsafe fn upper_nibble_mask() -> __m512i { + _mm512_loadu_si512(UPPER_NIBBLE_MASK_ARRAY.as_ptr().cast::()) +} + +#[target_feature(enable = "avx512f")] +#[inline] +pub(crate) unsafe fn commas_toggle_mask() -> __m512i { + _mm512_loadu_si512(COMMAS_TOGGLE_MASK_ARRAY.as_ptr().cast::()) +} + +#[target_feature(enable = "avx512f")] +#[inline] +pub(crate) unsafe fn colons_toggle_mask() -> __m512i { + _mm512_loadu_si512(COLON_TOGGLE_MASK_ARRAY.as_ptr().cast::()) +} + +#[target_feature(enable = "avx512f")] +#[inline] +pub(crate) unsafe fn colons_and_commas_toggle_mask() -> __m512i { + _mm512_or_si512(colons_toggle_mask(), commas_toggle_mask()) +} + +pub(crate) struct BlockClassifier512 { + upper_nibble_mask: __m512i, +} + +impl BlockClassifier512 { + #[target_feature(enable = "avx512f")] + #[inline] + pub(crate) unsafe fn new() -> Self { + Self { + upper_nibble_mask: upper_nibble_mask(), + } + } + + #[target_feature(enable = "avx512f")] + #[inline] + pub(crate) unsafe fn toggle_commas(&mut self) { + self.upper_nibble_mask = _mm512_xor_si512(self.upper_nibble_mask, commas_toggle_mask()); + } + + #[target_feature(enable = "avx512f")] + #[inline] + pub(crate) unsafe fn toggle_colons(&mut self) { + self.upper_nibble_mask = _mm512_xor_si512(self.upper_nibble_mask, colons_toggle_mask()); + } + + #[target_feature(enable = "avx512f")] + #[inline] + pub(crate) unsafe fn toggle_colons_and_commas(&mut self) { + self.upper_nibble_mask = _mm512_xor_si512(self.upper_nibble_mask, colons_and_commas_toggle_mask()); + } + + #[target_feature(enable = "avx512f")] + #[target_feature(enable = "avx512bw")] + #[inline] + pub(crate) unsafe fn classify_block(&self, block: &[u8]) -> BlockClassification512 { + let byte_vector = _mm512_loadu_si512(block.as_ptr().cast::()); + let shifted_byte_vector = _mm512_srli_epi16::<4>(byte_vector); + let upper_nibble_byte_vector = _mm512_and_si512(shifted_byte_vector, upper_nibble_zeroing_mask()); + let lower_nibble_lookup = _mm512_shuffle_epi8(lower_nibble_mask(), byte_vector); + let upper_nibble_lookup = _mm512_shuffle_epi8(self.upper_nibble_mask, upper_nibble_byte_vector); + let structural= _mm512_cmpeq_epi8_mask(lower_nibble_lookup, upper_nibble_lookup); + + BlockClassification512 { structural } + } +} + +pub(crate) struct BlockClassification512 { + pub(crate) structural: u64, +} diff --git a/crates/rsonpath-lib/src/classification/structural/shared/vector_neon.rs b/crates/rsonpath-lib/src/classification/structural/shared/vector_neon.rs new file mode 100644 index 00000000..7d544ec3 --- /dev/null +++ b/crates/rsonpath-lib/src/classification/structural/shared/vector_neon.rs @@ -0,0 +1,134 @@ +#[cfg(target_arch = "aarch64")] +use ::core::arch::aarch64::*; + +#[inline] +#[target_feature(enable = "neon")] +unsafe fn neon_movemask_epi8(cmp_vector: uint8x16_t) -> i16 { + // Tablica shiftów + let shift_values: [i8; 16] = [-7, -6, -5, -4, -3, -2, -1, 0, -7, -6, -5, -4, -3, -2, -1, 0]; + + // Załaduj shift do NEON + let vshift = vld1q_s8(shift_values.as_ptr()); + + // Zamaskowanie bitu 7 (0x80) + let vmask = vandq_u8(cmp_vector, vdupq_n_u8(0x80)); + + // Przesunięcie bitów + let vmask = vshlq_u8(vmask, vshift); + + // Sumowanie dolnej i górnej połowy + let low = vaddv_u8(vget_low_u8(vmask)) as i16; + let high = vaddv_u8(vget_high_u8(vmask)) as i16; + + low | (high << 8) +} + +#[inline] +#[target_feature(enable = "neon")] +unsafe fn neon_shuffle(data: int8x16_t, mask: int8x16_t) -> int8x16_t { + let low = vget_low_s8(data); + let high = vget_high_s8(data); + let recombined = int8x8x2_t(low, high); + + vcombine_s8( + vtbl2_s8(recombined, vand_s8(VDUP_N_S8(0x0F), vget_low_s8(mask))), + vtbl2_s8(recombined, vand_s8(VDUP_N_S8(0x0F), vget_high_s8(mask))) + ) +} + +const LOWER_NIBBLE_MASK_ARRAY: [u8; 32] = [ + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x03, 0x01, 0x02, 0x01, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x03, 0x01, 0x02, 0x01, 0xff, 0xff, +]; +const UPPER_NIBBLE_MASK_ARRAY: [u8; 32] = [ + 0xfe, 0xfe, 0x10, 0x10, 0xfe, 0x01, 0xfe, 0x01, 0xfe, 0xfe, 0xfe, 0xfe, 0xfe, 0xfe, 0xfe, 0xfe, 0xfe, 0xfe, 0x10, + 0x10, 0xfe, 0x01, 0xfe, 0x01, 0xfe, 0xfe, 0xfe, 0xfe, 0xfe, 0xfe, 0xfe, 0xfe, +]; +const COMMAS_TOGGLE_MASK_ARRAY: [u8; 32] = [ + 0x00, 0x00, 0x12, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x12, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, +]; +const COLON_TOGGLE_MASK_ARRAY: [u8; 32] = [ + 0x00, 0x00, 0x00, 0x13, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x13, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, +]; + +#[target_feature(enable = "neon")] +#[inline] +pub(crate) unsafe fn upper_nibble_zeroing_mask() -> int8x16_t { + vdupq_n_s8(0x0F) +} + +#[target_feature(enable = "neon")] +#[inline] +pub(crate) unsafe fn lower_nibble_mask() -> int8x16_t { + vreinterpretq_s8_u8(vld1q_u8(LOWER_NIBBLE_MASK_ARRAY.as_ptr())) +} + +#[target_feature(enable = "neon")] +#[inline] +pub(crate) unsafe fn commas_toggle_mask() -> int8x16_t { + vreinterpretq_s8_u8(vld1q_u8(COMMAS_TOGGLE_MASK_ARRAY.as_ptr())) +} + +#[target_feature(enable = "neon")] +#[inline] +pub(crate) unsafe fn colons_toggle_mask() -> int8x16_t { + vreinterpretq_s8_u8(vld1q_u8(COLON_TOGGLE_MASK_ARRAY.as_ptr())) +} + +#[target_feature(enable = "neon")] +#[inline] +pub(crate) unsafe fn colons_and_commas_toggle_mask() -> int8x16_t { + vorrq_s8(colons_toggle_mask(), commas_toggle_mask()) +} + +pub(crate) struct BlockClassifierNeon { + upper_nibble_mask: int8x16_t, +} + +impl BlockClassifierNeon { + #[target_feature(enable = "neon")] + #[inline] + pub(crate) unsafe fn new() -> Self { + Self { + upper_nibble_mask: vreinterpretq_s8_u8(vld1q_u8(UPPER_NIBBLE_MASK_ARRAY.as_ptr())), + } + } + + #[target_feature(enable = "neon")] + #[inline] + pub(crate) unsafe fn toggle_commas(&mut self) { + self.upper_nibble_mask = veorq_s8(self.upper_nibble_mask, commas_toggle_mask()); + } + + #[target_feature(enable = "neon")] + #[inline] + pub(crate) unsafe fn toggle_colons(&mut self) { + self.upper_nibble_mask = veorq_s8(self.upper_nibble_mask, colons_toggle_mask()); + } + + #[target_feature(enable = "neon")] + #[inline] + pub(crate) unsafe fn toggle_colons_and_commas(&mut self) { + self.upper_nibble_mask = veorq_s8(self.upper_nibble_mask, colons_and_commas_toggle_mask()); + } + + #[target_feature(enable = "neon")] + #[inline] + pub(crate) unsafe fn classify_block(&self, block: &[u8]) -> BlockClassification128 { + let byte_vector = vreinterpretq_s16_u8(vld1q_u8(block.as_ptr())); + let shifted_byte_vector = vreinterpretq_s8_s16(vshrq_n_s16(byte_vector, 4)); + let upper_nibble_byte_vector = vandq_s8(shifted_byte_vector, upper_nibble_zeroing_mask()); + let lower_nibble_lookup = neon_shuffle(lower_nibble_mask(), vreinterpretq_s8_s16(byte_vector)); + let upper_nibble_lookup = neon_shuffle(self.upper_nibble_mask, upper_nibble_byte_vector); + let structural_vector = vceqq_s8(lower_nibble_lookup, upper_nibble_lookup); + let structural = neon_movemask_epi8(structural_vector) as u16; + + BlockClassification128 { structural } + } +} + +pub(crate) struct BlockClassification128 { + pub(crate) structural: u16, +} diff --git a/crates/rsonpath-lib/src/engine/serde.rs b/crates/rsonpath-lib/src/engine/serde.rs new file mode 100644 index 00000000..24b2b1b1 --- /dev/null +++ b/crates/rsonpath-lib/src/engine/serde.rs @@ -0,0 +1,211 @@ +use crate::{ + automaton::Automaton, + engine::{main::MainEngine, Compiler}, +}; +use serde::{ + de::{self, Visitor}, + ser::SerializeTuple, + Deserialize, Serialize, +}; + +#[derive(Debug, Serialize, Deserialize)] +enum BinaryVersion { + /// Placeholder for any version in the past, used for tests. + Past, + /// Introduced binary serialization in v0.9.4. + V1, +} + +impl de::Expected for BinaryVersion { + fn fmt(&self, formatter: &mut std::fmt::Formatter) -> std::fmt::Result { + match *self { + Self::Past => write!(formatter, "Past"), + Self::V1 => write!(formatter, "v0.9.4"), + } + } +} + +impl Serialize for MainEngine { + #[inline] + fn serialize(&self, serializer: S) -> Result + where + S: serde::Serializer, + { + let mut tuple_ser = serializer.serialize_tuple(2)?; + tuple_ser.serialize_element(&BinaryVersion::V1)?; + tuple_ser.serialize_element(&self.automaton())?; + tuple_ser.end() + } +} + +impl<'de> Deserialize<'de> for MainEngine { + #[inline] + fn deserialize(deserializer: D) -> Result + where + D: serde::Deserializer<'de>, + { + let automaton = deserializer.deserialize_tuple(2, EngineVisitor)?; + Ok(Self::from_compiled_query(automaton)) + } +} + +struct EngineVisitor; + +impl<'de> Visitor<'de> for EngineVisitor { + type Value = Automaton; + + fn expecting(&self, formatter: &mut std::fmt::Formatter) -> std::fmt::Result { + write!(formatter, "the binary version and the Automaton") + } + + fn visit_seq(self, mut seq: A) -> Result + where + A: serde::de::SeqAccess<'de>, + { + let version = seq.next_element::()?; + match version { + Some(BinaryVersion::V1) => (), + Some(v) => return Err(de::Error::custom(format!("binary version {:?} is incompatible", v))), + None => return Err(de::Error::missing_field("version")), + } + let automaton = seq.next_element::()?; + match automaton { + Some(a) => Ok(a), + None => Err(de::Error::missing_field("automaton")), + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::{ + automaton::Automaton, + engine::{Compiler, RsonpathEngine}, + }; + use serde::{ser::SerializeTuple, Serialize, Serializer}; + use std::error::Error; + + #[test] + fn automaton_round_trip() -> Result<(), Box> { + let query_str = "$..phoneNumbers[*].number"; + let query = rsonpath_syntax::parse(query_str)?; + let automaton = Automaton::new(&query)?; + let engine = RsonpathEngine::from_compiled_query(automaton.clone()); + + let json_string = serde_json::to_string(&engine)?; + + let round_trip: RsonpathEngine = serde_json::from_str(&json_string)?; + + assert_eq!(&automaton, round_trip.automaton()); + + Ok(()) + } + + #[test] + fn deserializing_from_older_version() -> Result<(), Box> { + struct OldEngine { + automaton: Automaton, + } + impl Serialize for OldEngine { + #[inline] + fn serialize(&self, serializer: S) -> Result + where + S: Serializer, + { + let mut tuple_ser = serializer.serialize_tuple(2)?; + tuple_ser.serialize_element(&BinaryVersion::Past)?; + tuple_ser.serialize_element(&self.automaton)?; + tuple_ser.end() + } + } + + let query_str = "$..phoneNumbers[*].number"; + let query = rsonpath_syntax::parse(query_str)?; + let automaton = Automaton::new(&query)?; + let engine = OldEngine { automaton }; + + let json_string = serde_json::to_string(&engine)?; + + match serde_json::from_str::(&json_string) { + Ok(_) => panic!("expected error"), + Err(e) => assert!(e.to_string().contains("binary version Past is incompatible")), + } + + Ok(()) + } + + mod proptests { + use super::{Automaton, Compiler, RsonpathEngine}; + use pretty_assertions::assert_eq; + use proptest::prelude::*; + use rsonpath_syntax_proptest::{ArbitraryJsonPathQuery, ArbitraryJsonPathQueryParams}; + + proptest! { + #[test] + fn main_engine_cbor_roundtrips(ArbitraryJsonPathQuery { parsed, .. } in prop::arbitrary::any_with::( + ArbitraryJsonPathQueryParams { + only_rsonpath_supported_subset: true, + ..Default::default() + } + )) { + use std::io; + struct ReadBuf<'a> { + buf: &'a [u8], + idx: usize, + } + impl<'a> io::Read for &mut ReadBuf<'a> { + fn read(&mut self, buf: &mut [u8]) -> Result { + let len = std::cmp::min(self.buf.len() - self.idx, buf.len()); + buf.copy_from_slice(&self.buf[self.idx..self.idx + len]); + self.idx += len; + Ok(len) + } + } + + let automaton = Automaton::new(&parsed)?; + let engine = RsonpathEngine::from_compiled_query(automaton.clone()); + + let mut buf = vec![]; + ciborium::into_writer(&engine, &mut buf)?; + + let mut read = ReadBuf { buf: &buf, idx: 0 }; + let engine_deser = ciborium::from_reader::(&mut read)?; + + assert_eq!(&automaton, engine_deser.automaton()); + } + + #[test] + fn main_engine_json_roundtrips(ArbitraryJsonPathQuery { parsed, .. } in prop::arbitrary::any_with::( + ArbitraryJsonPathQueryParams { + only_rsonpath_supported_subset: true, + ..Default::default() + } + )) { + let automaton = Automaton::new(&parsed)?; + let engine = RsonpathEngine::from_compiled_query(automaton.clone()); + + let json_str = serde_json::to_string(&engine)?; + let engine_deser = serde_json::from_str::(&json_str)?; + + assert_eq!(&automaton, engine_deser.automaton()); + } + + #[test] + fn main_engine_message_pack_roundtrips(ArbitraryJsonPathQuery { parsed, .. } in prop::arbitrary::any_with::( + ArbitraryJsonPathQueryParams { + only_rsonpath_supported_subset: true, + ..Default::default() + } + )) { + let automaton = Automaton::new(&parsed)?; + let engine = RsonpathEngine::from_compiled_query(automaton.clone()); + + let buf = rmp_serde::to_vec(&engine)?; + let engine_deser = rmp_serde::from_slice::(&buf)?; + + assert_eq!(&automaton, engine_deser.automaton()); + } + } + } +} diff --git a/crates/rsonpath-lib/src/lib.rs b/crates/rsonpath-lib/src/lib.rs index f2784c13..e356fbfa 100644 --- a/crates/rsonpath-lib/src/lib.rs +++ b/crates/rsonpath-lib/src/lib.rs @@ -189,6 +189,7 @@ warn(clippy::print_stderr, clippy::print_stdout, clippy::todo) )] #![cfg_attr(docsrs, feature(doc_cfg))] +#![feature(avx512_target_feature)] pub mod automaton; pub mod classification; diff --git a/crates/rsonpath-lib/src/string_pattern.rs b/crates/rsonpath-lib/src/string_pattern.rs new file mode 100644 index 00000000..f6a9c688 --- /dev/null +++ b/crates/rsonpath-lib/src/string_pattern.rs @@ -0,0 +1,75 @@ +use rsonpath_syntax::str::JsonString; + +/// String pattern coming from a JSONPath query that can be matched against strings in a JSON. +/// +/// Right now the only pattern is matching against a given [`JsonString`]. +#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))] +#[derive(Debug, Clone)] +pub struct StringPattern(JsonString); + +impl std::hash::Hash for StringPattern { + #[inline] + fn hash(&self, state: &mut H) { + self.0.hash(state); + } +} + +impl PartialOrd for StringPattern { + #[inline] + fn partial_cmp(&self, other: &Self) -> Option { + Some(self.0.unquoted().cmp(other.0.unquoted())) + } +} + +impl Ord for StringPattern { + #[inline] + fn cmp(&self, other: &Self) -> std::cmp::Ordering { + self.0.unquoted().cmp(other.0.unquoted()) + } +} + +impl PartialEq for StringPattern { + #[inline] + fn eq(&self, other: &Self) -> bool { + self.0 == other.0 + } +} + +impl Eq for StringPattern {} + +impl StringPattern { + /// Get the underlying [`JsonString`] as bytes, including the delimiting double quote symbols. + #[inline] + #[must_use] + pub fn quoted(&self) -> &[u8] { + self.0.quoted().as_bytes() + } + + /// Get the underlying [`JsonString`] as bytes, without the delimiting quotes. + #[inline] + #[must_use] + pub fn unquoted(&self) -> &[u8] { + self.0.unquoted().as_bytes() + } + + /// Create a new pattern from a given [`JsonString`]. + #[inline] + #[must_use] + pub fn new(string: &JsonString) -> Self { + Self(string.clone()) + } +} + +impl From for StringPattern { + #[inline(always)] + fn from(value: JsonString) -> Self { + Self::new(&value) + } +} + +impl From<&JsonString> for StringPattern { + #[inline(always)] + fn from(value: &JsonString) -> Self { + Self::new(value) + } +} diff --git a/crates/rsonpath-lib/tests/engine_serialization_snapshots.rs b/crates/rsonpath-lib/tests/engine_serialization_snapshots.rs new file mode 100644 index 00000000..8fa71e52 --- /dev/null +++ b/crates/rsonpath-lib/tests/engine_serialization_snapshots.rs @@ -0,0 +1,42 @@ +#[cfg(feature = "serde")] +mod ron { + use insta::assert_ron_snapshot; + use rsonpath::engine::{Compiler, RsonpathEngine}; + use std::error::Error; + + fn engine(string: &str) -> Result> { + let query = rsonpath_syntax::parse(string)?; + let engine = RsonpathEngine::compile_query(&query)?; + Ok(engine) + } + + #[test] + fn empty_query() -> Result<(), Box> { + assert_ron_snapshot!(&engine("$")?); + Ok(()) + } + + #[test] + fn readme_query() -> Result<(), Box> { + assert_ron_snapshot!(&engine("$.jsonpath[*]")?); + Ok(()) + } + + #[test] + fn jsonpath_example_query() -> Result<(), Box> { + assert_ron_snapshot!(&engine("$..phoneNumbers[*].number")?); + Ok(()) + } + + #[test] + fn real_life_query() -> Result<(), Box> { + assert_ron_snapshot!(&engine("$.personal.details.contact.information.phones.home")?); + Ok(()) + } + + #[test] + fn slice() -> Result<(), Box> { + assert_ron_snapshot!(&engine("$..entries[3:5:7]")?); + Ok(()) + } +} diff --git a/crates/rsonpath-lib/tests/snapshots/engine_serialization_snapshots__ron__empty_query.snap b/crates/rsonpath-lib/tests/snapshots/engine_serialization_snapshots__ron__empty_query.snap new file mode 100644 index 00000000..7b6e463d --- /dev/null +++ b/crates/rsonpath-lib/tests/snapshots/engine_serialization_snapshots__ron__empty_query.snap @@ -0,0 +1,21 @@ +--- +source: crates/rsonpath-lib/tests/engine_serialization_snapshots.rs +expression: "&engine(\"$\")?" +snapshot_kind: text +--- +(V1, Automaton( + states: [ + StateTable( + attributes: StateAttributes(2), + member_transitions: [], + array_transitions: [], + fallback_state: State(0), + ), + StateTable( + attributes: StateAttributes(1), + member_transitions: [], + array_transitions: [], + fallback_state: State(0), + ), + ], +)) diff --git a/crates/rsonpath-lib/tests/snapshots/engine_serialization_snapshots__ron__jsonpath_example_query.snap b/crates/rsonpath-lib/tests/snapshots/engine_serialization_snapshots__ron__jsonpath_example_query.snap new file mode 100644 index 00000000..f9042dd5 --- /dev/null +++ b/crates/rsonpath-lib/tests/snapshots/engine_serialization_snapshots__ron__jsonpath_example_query.snap @@ -0,0 +1,84 @@ +--- +source: crates/rsonpath-lib/tests/engine_serialization_snapshots.rs +expression: "&engine(\"$..phoneNumbers[*].number\")?" +snapshot_kind: text +--- +(V1, Automaton( + states: [ + StateTable( + attributes: StateAttributes(2), + member_transitions: [], + array_transitions: [], + fallback_state: State(0), + ), + StateTable( + attributes: StateAttributes(0), + member_transitions: [ + (StringPattern(JsonString( + quoted: "\"phoneNumbers\"", + )), State(2)), + ], + array_transitions: [], + fallback_state: State(1), + ), + StateTable( + attributes: StateAttributes(0), + member_transitions: [ + (StringPattern(JsonString( + quoted: "\"phoneNumbers\"", + )), State(4)), + ], + array_transitions: [], + fallback_state: State(3), + ), + StateTable( + attributes: StateAttributes(8), + member_transitions: [ + (StringPattern(JsonString( + quoted: "\"phoneNumbers\"", + )), State(2)), + (StringPattern(JsonString( + quoted: "\"number\"", + )), State(6)), + ], + array_transitions: [], + fallback_state: State(1), + ), + StateTable( + attributes: StateAttributes(8), + member_transitions: [ + (StringPattern(JsonString( + quoted: "\"phoneNumbers\"", + )), State(4)), + (StringPattern(JsonString( + quoted: "\"number\"", + )), State(5)), + ], + array_transitions: [], + fallback_state: State(3), + ), + StateTable( + attributes: StateAttributes(9), + member_transitions: [ + (StringPattern(JsonString( + quoted: "\"phoneNumbers\"", + )), State(2)), + (StringPattern(JsonString( + quoted: "\"number\"", + )), State(6)), + ], + array_transitions: [], + fallback_state: State(1), + ), + StateTable( + attributes: StateAttributes(1), + member_transitions: [ + (StringPattern(JsonString( + quoted: "\"phoneNumbers\"", + )), State(2)), + ], + array_transitions: [], + fallback_state: State(1), + ), + ], +)) diff --git a/crates/rsonpath-lib/tests/snapshots/engine_serialization_snapshots__ron__readme_query.snap b/crates/rsonpath-lib/tests/snapshots/engine_serialization_snapshots__ron__readme_query.snap new file mode 100644 index 00000000..e40ccafb --- /dev/null +++ b/crates/rsonpath-lib/tests/snapshots/engine_serialization_snapshots__ron__readme_query.snap @@ -0,0 +1,37 @@ +--- +source: crates/rsonpath-lib/tests/engine_serialization_snapshots.rs +expression: "&engine(\"$.jsonpath[*]\")?" +snapshot_kind: text +--- +(V1, Automaton( + states: [ + StateTable( + attributes: StateAttributes(2), + member_transitions: [], + array_transitions: [], + fallback_state: State(0), + ), + StateTable( + attributes: StateAttributes(4), + member_transitions: [ + (StringPattern(JsonString( + quoted: "\"jsonpath\"", + )), State(2)), + ], + array_transitions: [], + fallback_state: State(0), + ), + StateTable( + attributes: StateAttributes(8), + member_transitions: [], + array_transitions: [], + fallback_state: State(3), + ), + StateTable( + attributes: StateAttributes(1), + member_transitions: [], + array_transitions: [], + fallback_state: State(0), + ), + ], +)) diff --git a/crates/rsonpath-lib/tests/snapshots/engine_serialization_snapshots__ron__real_life_query.snap b/crates/rsonpath-lib/tests/snapshots/engine_serialization_snapshots__ron__real_life_query.snap new file mode 100644 index 00000000..29240077 --- /dev/null +++ b/crates/rsonpath-lib/tests/snapshots/engine_serialization_snapshots__ron__real_life_query.snap @@ -0,0 +1,81 @@ +--- +source: crates/rsonpath-lib/tests/engine_serialization_snapshots.rs +expression: "&engine(\"$.personal.details.contact.information.phones.home\")?" +snapshot_kind: text +--- +(V1, Automaton( + states: [ + StateTable( + attributes: StateAttributes(2), + member_transitions: [], + array_transitions: [], + fallback_state: State(0), + ), + StateTable( + attributes: StateAttributes(4), + member_transitions: [ + (StringPattern(JsonString( + quoted: "\"personal\"", + )), State(2)), + ], + array_transitions: [], + fallback_state: State(0), + ), + StateTable( + attributes: StateAttributes(4), + member_transitions: [ + (StringPattern(JsonString( + quoted: "\"details\"", + )), State(3)), + ], + array_transitions: [], + fallback_state: State(0), + ), + StateTable( + attributes: StateAttributes(4), + member_transitions: [ + (StringPattern(JsonString( + quoted: "\"contact\"", + )), State(4)), + ], + array_transitions: [], + fallback_state: State(0), + ), + StateTable( + attributes: StateAttributes(4), + member_transitions: [ + (StringPattern(JsonString( + quoted: "\"information\"", + )), State(5)), + ], + array_transitions: [], + fallback_state: State(0), + ), + StateTable( + attributes: StateAttributes(4), + member_transitions: [ + (StringPattern(JsonString( + quoted: "\"phones\"", + )), State(6)), + ], + array_transitions: [], + fallback_state: State(0), + ), + StateTable( + attributes: StateAttributes(12), + member_transitions: [ + (StringPattern(JsonString( + quoted: "\"home\"", + )), State(7)), + ], + array_transitions: [], + fallback_state: State(0), + ), + StateTable( + attributes: StateAttributes(1), + member_transitions: [], + array_transitions: [], + fallback_state: State(0), + ), + ], +)) diff --git a/crates/rsonpath-lib/tests/snapshots/engine_serialization_snapshots__ron__slice.snap b/crates/rsonpath-lib/tests/snapshots/engine_serialization_snapshots__ron__slice.snap new file mode 100644 index 00000000..212a92ec --- /dev/null +++ b/crates/rsonpath-lib/tests/snapshots/engine_serialization_snapshots__ron__slice.snap @@ -0,0 +1,50 @@ +--- +source: crates/rsonpath-lib/tests/engine_serialization_snapshots.rs +expression: "&engine(\"$..entries[3:5:7]\")?" +snapshot_kind: text +--- +(V1, Automaton( + states: [ + StateTable( + attributes: StateAttributes(2), + member_transitions: [], + array_transitions: [], + fallback_state: State(0), + ), + StateTable( + attributes: StateAttributes(0), + member_transitions: [ + (StringPattern(JsonString( + quoted: "\"entries\"", + )), State(2)), + ], + array_transitions: [], + fallback_state: State(1), + ), + StateTable( + attributes: StateAttributes(56), + member_transitions: [ + (StringPattern(JsonString( + quoted: "\"entries\"", + )), State(2)), + ], + array_transitions: [ + ArrayTransition( + label: Index(JsonUInt(3)), + target: State(3), + ), + ], + fallback_state: State(1), + ), + StateTable( + attributes: StateAttributes(1), + member_transitions: [ + (StringPattern(JsonString( + quoted: "\"entries\"", + )), State(2)), + ], + array_transitions: [], + fallback_state: State(1), + ), + ], +)) diff --git a/rust-toolchain.toml b/rust-toolchain.toml index cf6a8794..4ccfba24 100644 --- a/rust-toolchain.toml +++ b/rust-toolchain.toml @@ -1,3 +1,3 @@ [toolchain] -channel = "stable" +channel = "nightly" components = [ "cargo", "rustfmt", "clippy" ] \ No newline at end of file