Fix unicode support in PCRE2

(cherry picked from commit 6bdad27)

Author: Pavel Marek

com.oracle.truffle.r.native/fficall/src/truffle_common/pcre2_rffi.c

@@ -26,7 +26,7 @@
  * For readability, the file first lists the declarations of the functions and then their implementation.
  * We use libpcre2-8, which is a version of the library that uses 8-bit character width (uint8_t).
  * The pcre2 header uses some defines for its types, we rather use standard types and explicitly enumerate
- * the type mappings in the following paragraph:
+ * the type mappings in the following table:
  * PCRE2_SIZE ... size_t
  * PCRE2_SPTR ... uint8_t *
  * PCRE2_UCHAR ... uint8_t
@@ -41,29 +41,24 @@
 // Define for printing important PCRE2-specific function calls.
 //#define FASTR_PCRE2_DEBUG
 
+// Typedefs for callbacks from native into Java.
+typedef void (*match_cb_t)(size_t start_idx, size_t end_idx);
+typedef void (*capture_cb_t)(size_t capture_idx, size_t start_idx, size_t end_idx);
+typedef void (*set_capture_name_cb_t)(const char *name, int index);
+
 pcre2_code *call_pcre2_compile(char *pattern, uint32_t options, int *error_code, int *error_offset);
 uint32_t call_pcre2_capture_count(pcre2_code *re);
 /**
  * Returns the count of all the named captures. A named capture is also a capture, so the number
  * returned by this function is always lower than the number returned by `call_pcre2_capture_count`.
  */
 uint32_t call_pcre2_names_count(pcre2_code *re);
-int call_pcre2_get_capture_names(void (*set_capture_name_cb)(const char *name, int index), pcre2_code *re);
+int call_pcre2_get_capture_names(set_capture_name_cb_t set_capture_name_cb, pcre2_code *re);
 
 /**
  * The implementation is heavily inspired by the demo application provided on the official
  * PCRE2 website and by the implementation of`do_regexpr` in GNU-R.
  *
- * The implementation does the first match, and then continues iff `stop_after_first_match`
- * is 1. The code that analyses the match data from PCRE2 library (two for loops) seems duplicated,
- * but this is how the pcre2demo is structured.
- *
- * The given callbacks are called (reported) in appropriate time. Note that some of the capture
- * may be reported twice - once without a name and once with name. Generally, there may be
- * reported duplicate captures. A capture is uniquelly identified by its start index and end index.
- * Every named capture is also a capture, so the number of named captures is always lower than
- * the number of all the captures.
- *
  * @param match_cb A function (callback) that is called when a match occured.
  * @param capture_cb A function that is called once a match for some capture occured.
  *     If there are no captures in the pattern, this callback is never called.
@@ -74,18 +69,25 @@ int call_pcre2_get_capture_names(void (*set_capture_name_cb)(const char *name, i
  * @param options An option bitset. See pcre2.h.
  * @param stop_after_first_match If 1, only first match is done.
  *
- * @returns Number of matches, or -1 on an error.
+ * @returns Number of matches, or PCRE2 error code (negative integer) on error. To convert
+ *    the error code into string, call `call_pcre2_errcode_to_string`.
  */
 int call_pcre2_match(
-    void (*match_cb)(size_t start_idx, size_t end_idx),
-    void (*capture_cb)(size_t capture_idx, size_t start_idx, size_t end_idx),
+    match_cb_t match_cb,
+    capture_cb_t capture_cb,
     pcre2_code *re,
     uint8_t *subject,
     uint32_t options,
     int stop_after_first_match
 );
-
 void call_pcre2_pattern_free(pcre2_code *compiled_pattern);
+void call_pcre2_errcode_to_string(int errcode, char *buff, size_t buff_len);
+
+// Helper functions
+static int is_valid_index(size_t index);
+static int is_utf8_continuation_byte(uint8_t byte);
+static void report_captures(capture_cb_t capture_cb, uint32_t capture_count, const size_t *ovector);
+static size_t advance_offset(size_t offset, int utf8, const uint8_t *subject, size_t subject_len);
 
 
 // TODO: error_offset should be `size_t`.
@@ -126,15 +128,6 @@ uint32_t call_pcre2_names_count(pcre2_code *re)
     return name_count;
 }
 
-/**
- * In some corner cases, like nested captures, PCRE2 returns invalid indexes.
- * It is unnecessary to report these invalid indexes to Java.
- */
-static int is_valid_index(size_t index)
-{
-    return index != ((size_t) -1);
-}
-
 int call_pcre2_get_capture_names(void (*set_capture_name_cb)(const char *name, int index), pcre2_code *re)
 {
     uint32_t names_count = call_pcre2_names_count(re);
@@ -161,51 +154,34 @@ int call_pcre2_get_capture_names(void (*set_capture_name_cb)(const char *name, i
 }
 
 int call_pcre2_match(
-    void (*match_cb)(size_t start_idx, size_t end_idx),
-    void (*capture_cb)(size_t capture_idx, size_t start_idx, size_t end_idx),
+    match_cb_t match_cb,
+    capture_cb_t capture_cb,
     pcre2_code *re,
     uint8_t *subject,
-    uint32_t options,
+    uint32_t first_match_options,
     int stop_after_first_match
 )
 {
-    size_t subject_len = strlen((char *)subject);
-    //uint8_t *subject = (uint8_t *)subject_str;
-
-#ifdef FASTR_PCRE2_DEBUG
-    printf("call_pcre2_match: subject (len=%u) = [", subject_len);
-    for (size_t i = 0; i < subject_len; i++) {
-        printf("%d, ", subject[i]);
-    }
-    printf("]\n");
-#endif
-
+    size_t subject_len = strlen((char *) subject);
     pcre2_match_data *match_data = pcre2_match_data_create_from_pattern(re, NULL);
-
-    // Check for named captures
     uint32_t capture_count = call_pcre2_capture_count(re);
-    uint32_t names_count = call_pcre2_names_count(re);
-    // Named group implies capture group.
-    if (!(capture_count >= names_count)) {
-        fatalError("[pcre2_rffi]: Unexpected Capture_count < names_count");
-    }
-    uint8_t *name_table = NULL;
-    uint32_t name_entry_size = 0;
 
-    if (names_count > 0) {
-        pcre2_pattern_info(re, PCRE2_INFO_NAMETABLE, &name_table);
-        pcre2_pattern_info(re, PCRE2_INFO_NAMEENTRYSIZE, &name_entry_size);
-    }
+    uint32_t pattern_option_bits = 0;
+    (void)pcre2_pattern_info(re, PCRE2_INFO_ALLOPTIONS, &pattern_option_bits);
+    int utf8 = (pattern_option_bits & PCRE2_UTF) != 0;
+#ifdef FASTR_PCRE2_DEBUG
+    printf("call_pcre2_match: utf8 option = %d\n", utf8);
+#endif
 
     int match_count = 0;
     // We use the default match context, and subject_offset is 0.
     // rc corresponds to the count of captured groups plus one, or error code if rc is negative.
-    int rc = pcre2_match(re, subject, subject_len, 0, options, match_data, NULL);
+    int rc = pcre2_match(re, subject, subject_len, 0, first_match_options, match_data, NULL);
     if (rc == PCRE2_ERROR_NOMATCH) {
         return 0;
     } else if (rc < 0) {
-        // TODO: fatalError("pcre2_match rc < 0");
-        return -1;
+        pcre2_match_data_free(match_data);
+        return rc;
     } else {
         match_count++;
     }
@@ -218,51 +194,47 @@ int call_pcre2_match(
     size_t *ovector = pcre2_get_ovector_pointer(match_data);
 
     if (ovector[0] > ovector[1]) {
-        printf("Match Error: \\K special case\n");
+#ifdef FASTR_PCRE2_DEBUG
+        printf("call_pcre2_match: \\K special case\n");
+#endif
+        // \K special case
+        pcre2_match_data_free(match_data);
         return -1;
     }
 
 #ifdef FASTR_PCRE2_DEBUG
-    printf("call_pcre2_match: match_cb(%u, %u)\n", ovector[0], ovector[1]);
+    printf("call_pcre2_match: match_cb(%lu, %lu)\n", ovector[0], ovector[1]);
 #endif
     match_cb(ovector[0], ovector[1]);
-
-    // The rest of the ovector is for the capture groups only.
-    for (size_t i = 0; i < capture_count; i++) {
-        size_t ovector_idx = i + 1;
-        int capture_idx = (int) i;
-        if (capture_count <= 0) {
-            fatalError("[pcre2_rffi.c]: capture_count <= 0");
-        }
-        size_t capt_start_idx = ovector[2 * ovector_idx];
-        size_t capt_end_idx = ovector[2 * ovector_idx + 1];
-        // We want to report only "valid" indexes to Java.
-        if (is_valid_index(capt_start_idx) && is_valid_index(capt_end_idx)) {
-#ifdef FASTR_PCRE2_DEBUG
-            printf("call_pcre2_match: capture_cb(%d, %u, %u)\n", capture_idx, capt_start_idx, capt_end_idx);
-#endif
-            capture_cb(capture_idx, capt_start_idx, capt_end_idx);
-        }
-    }
+    report_captures(capture_cb, capture_count, ovector);
 
     if (stop_after_first_match) {
         // The case for match_count == 0 was already processed.
         if (match_count != 1) {
-            fatalError("[pcre2_rffi.c]: match_count != 1");
+            fatalError("pcre2_rffi.c: match_count != 1");
         }
+        pcre2_match_data_free(match_data);
         return match_count;
     }
 
     // Find the rest of all the matches
     while (1) {
+        uint32_t options = 0;
         // start_offset is an index into subject from where to start next match.
         size_t start_offset = ovector[1];
-        uint32_t options = 0;
+        if (start_offset > subject_len) {
+#ifdef FASTR_PCRE2_DEBUG
+            printf("start_offset > subject_len\n");
+#endif
+            pcre2_match_data_free(match_data);
+            return match_count;
+        }
 
         // Check if the previous match was for an empty string.
         if (ovector[0] == ovector[1]) {
             if (ovector[0] == subject_len) {
                 // We are at the end of the subject.
+                pcre2_match_data_free(match_data);
                 return match_count;
             } else {
                 // We set the options here so that we prevent an infinite recursion.
@@ -271,16 +243,25 @@ int call_pcre2_match(
         } else {
             // Previous match was not an empty string.
             size_t prev_match_start_idx = pcre2_get_startchar(match_data);
+            // Handle \K special case
             if (start_offset <= prev_match_start_idx) {
-                if (prev_match_start_idx >= subject_len) {
+                // We have to increase start_offset
+#ifdef FASTR_PCRE2_DEBUG
+                printf("!! start_offset <= prev_match_start_idx\n");
+#endif
+                if (subject_len <= prev_match_start_idx) {
+                    // Reached end of subject.
+                    pcre2_match_data_free(match_data);
                     return match_count;
                 } else {
-                    // Advance by one character.
-                    start_offset = prev_match_start_idx + 1;
+                    start_offset = advance_offset(prev_match_start_idx, utf8, subject, subject_len);
                 }
             }
         }
 
+#ifdef FASTR_PCRE2_DEBUG
+        printf("call_pcre2_match: Calling pcre2_match(start_offset=%lu, options=%u)\n", start_offset, options);
+#endif
         // This time, we call `pcre2_match` with a specific offset into the subject.
         rc = pcre2_match(re, subject, subject_len, start_offset, options, match_data, NULL);
 
@@ -289,15 +270,19 @@ int call_pcre2_match(
             if (options == 0) {
                 return match_count;
             } else {
-                // Advance one code unit.
-                ovector[1] = start_offset + 1;
+                // Options != 0 means that the previous match was for an empty string.
+                ovector[1] = advance_offset(start_offset, utf8, subject, subject_len);
             }
             continue;
+        } else if (rc == PCRE2_ERROR_BADUTFOFFSET) {
+            // We provided pcre2_match function with bad offset into an UTF-8 character.
+            pcre2_match_data_free(match_data);
+            fatalError("pcre2_rffi.c: BADUTFOFFSET - should not happen");
+            return -1;
         } else if (rc < 0) {
             // This error is not recoverable
-            printf("Matching error %d\n", rc);
             pcre2_match_data_free(match_data);
-            return -1;
+            return rc;
         } else {
             match_count++;
         }
@@ -308,31 +293,73 @@ int call_pcre2_match(
             return -1;
         }
 
-#ifdef FASTR_PCRE2_DEBUG
-    printf("call_pcre2_match: match_cb(%u, %u)\n", ovector[0], ovector[1]);
-#endif
         match_cb(ovector[0], ovector[1]);
-
-        for (size_t i = 0; i < capture_count; i++) {
-            size_t ovector_idx = i + 1;
-            int capture_idx = (int) i;
-            if (capture_count <= 0) {
-                fatalError("[pcre2_rffi.c]: capture_count <= 0");
-            }
-            size_t capt_start_idx = ovector[2 * ovector_idx];
-            size_t capt_end_idx = ovector[2 * ovector_idx + 1];
-            if (is_valid_index(capt_start_idx) && is_valid_index(capt_end_idx)) {
-#ifdef FASTR_PCRE2_DEBUG
-                printf("call_pcre2_match: capture_cb(%d, %u, %u)\n", capture_idx, capt_start_idx, capt_end_idx);
-#endif
-                capture_cb(capture_idx, capt_start_idx, capt_end_idx);
-            }
-        }
+        report_captures(capture_cb, capture_count, ovector);
     }
+    pcre2_match_data_free(match_data);
     return match_count;
 }
 
 void call_pcre2_pattern_free(pcre2_code *compiled_pattern)
 {
     pcre2_code_free(compiled_pattern);
 }
+
+void call_pcre2_errcode_to_string(int errcode, char *buff, size_t buff_len)
+{
+    if (errcode >= 0) {
+        fatalError("pcre2_rffi.c: errcode >= 0");
+    }
+    int rc = pcre2_get_error_message(errcode, (uint8_t *)buff, buff_len);
+    if (rc < 0) {
+        printf("Fatal error: pcre2_get_error_message returned %d\n", rc);
+        exit(1);
+    }
+}
+
+/**
+ * In some corner cases, like nested captures, PCRE2 returns invalid indexes.
+ * It is unnecessary to report these invalid indexes to Java.
+ */
+static int is_valid_index(size_t index)
+{
+    return index != ((size_t) -1);
+}
+
+/**
+ * Returns true if the given byte is a continuation byte of some Unicode string, i.e. if it is
+ * not a start of some Unicode character.
+ */
+static int is_utf8_continuation_byte(uint8_t byte)
+{
+    return (byte & 0xc0) == 0x80;
+}
+
+static void report_captures(capture_cb_t capture_cb, uint32_t capture_count, const size_t *ovector)
+{
+    for (size_t i = 0; i < capture_count; i++) {
+        size_t ovector_idx = i + 1;
+        int capture_idx = (int) i;
+        if (capture_count <= 0) {
+            fatalError("capture_count <= 0");
+        }
+        size_t capt_start_idx = ovector[2 * ovector_idx];
+        size_t capt_end_idx = ovector[2 * ovector_idx + 1];
+        // We want to report only "valid" indexes to Java.
+        if (is_valid_index(capt_start_idx) && is_valid_index(capt_end_idx)) {
+            capture_cb(capture_idx, capt_start_idx, capt_end_idx);
+        }
+    }
+}
+
+static size_t advance_offset(size_t offset, int utf8, const uint8_t *subject, size_t subject_len)
+{
+    size_t next_offset = offset + 1;
+    if (utf8) {
+        while (is_utf8_continuation_byte(subject[next_offset]) && next_offset < subject_len) {
+            next_offset++;
+        }
+    }
+    return next_offset;
+}
+

com.oracle.truffle.r.nodes.test/src/com/oracle/truffle/r/nodes/test/PCRE2Tests.java

@@ -155,6 +155,9 @@ public Builder expectedCaptureNames(String[] captureNames) {
                     expectedCaptureIndexes(0, new int[]{0, 3}).
                     expectedCaptureNames(new String[]{null}).
                     build(),
+            TestData.builder().pattern(".*").subject("X")
+                    .expectedMatchIndexes(new int[]{0,1, 1, 1})
+                    .build(),
             TestData.builder().pattern("(?P<word>[a-z]+)").subject("abc123").
                     expectedMatchIndexes(new int[]{0, 3}).
                     expectedCaptureIndexes(0, new int[]{0, 3}).
@@ -221,6 +224,13 @@ public Builder expectedCaptureNames(String[] captureNames) {
                     expectedCaptureIndexes(1, new int[]{0, 0}).
                     expectedCaptureNames(new String[]{null, null}).
                     build(),
+            // Both pattern and subject are Unicode characters
+            TestData.builder().pattern("[⚽]").subject("─")
+                    .expectedMatchIndexes(new int[]{})
+                    .build(),
+            TestData.builder().pattern("[⚽]").subject("a")
+                    .expectedMatchIndexes(new int[]{})
+                    .build(),
     };
     // @formatter:on