Wire up options through the FFI API

bin/lex

@@ -29,7 +29,7 @@ ripper =
     []
   end
 
-prism = Prism.lex_compat(source, filepath)
+prism = Prism.lex_compat(source, filepath: filepath)
 prism_new = Prism.lex(source, filepath: filepath)
 if prism.errors.any?
   puts "Errors lexing:"

ext/prism/extension.c

@@ -268,7 +268,7 @@ dump_file(int argc, VALUE *argv, VALUE self) {
     extract_options(&options, filepath, keywords);
 
     pm_string_t input;
-    if (!pm_string_mapped_init(&input, options.filepath)) {
+    if (!pm_string_mapped_init(&input, (const char *) pm_string_source(&options.filepath))) {
         pm_options_free(&options);
         return Qnil;
     }
@@ -561,7 +561,7 @@ lex_file(int argc, VALUE *argv, VALUE self) {
     extract_options(&options, filepath, keywords);
 
     pm_string_t input;
-    if (!pm_string_mapped_init(&input, options.filepath)) {
+    if (!pm_string_mapped_init(&input, (const char *) pm_string_source(&options.filepath))) {
         pm_options_free(&options);
         return Qnil;
     }
@@ -672,7 +672,7 @@ parse_file(int argc, VALUE *argv, VALUE self) {
     extract_options(&options, filepath, keywords);
 
     pm_string_t input;
-    if (!pm_string_mapped_init(&input, options.filepath)) {
+    if (!pm_string_mapped_init(&input, (const char *) pm_string_source(&options.filepath))) {
         pm_options_free(&options);
         return Qnil;
     }
@@ -746,7 +746,7 @@ parse_file_comments(int argc, VALUE *argv, VALUE self) {
     extract_options(&options, filepath, keywords);
 
     pm_string_t input;
-    if (!pm_string_mapped_init(&input, options.filepath)) {
+    if (!pm_string_mapped_init(&input, (const char *) pm_string_source(&options.filepath))) {
         pm_options_free(&options);
         return Qnil;
     }
@@ -815,7 +815,7 @@ parse_lex_file(int argc, VALUE *argv, VALUE self) {
     extract_options(&options, filepath, keywords);
 
     pm_string_t input;
-    if (!pm_string_mapped_init(&input, options.filepath)) {
+    if (!pm_string_mapped_init(&input, (const char *) pm_string_source(&options.filepath))) {
         pm_options_free(&options);
         return Qnil;
     }
@@ -913,30 +913,6 @@ profile_file(VALUE self, VALUE filepath) {
     return Qnil;
 }
 
-/**
- * call-seq:
- *   Debug::parse_serialize_file_metadata(filepath, metadata) -> dumped
- *
- * Parse the file and serialize the result. This is mostly used to test this
- * path since it is used by client libraries.
- */
-static VALUE
-parse_serialize_file_metadata(VALUE self, VALUE filepath, VALUE metadata) {
-    pm_string_t input;
-    pm_buffer_t buffer;
-    pm_buffer_init(&buffer);
-
-    const char *checked = check_string(filepath);
-    if (!pm_string_mapped_init(&input, checked)) return Qnil;
-
-    pm_parse_serialize(pm_string_source(&input), pm_string_length(&input), &buffer, check_string(metadata));
-    VALUE result = rb_str_new(pm_buffer_value(&buffer), pm_buffer_length(&buffer));
-
-    pm_string_free(&input);
-    pm_buffer_free(&buffer);
-    return result;
-}
-
 /**
  * call-seq:
  *   Debug::inspect_node(source) -> inspected
@@ -1039,7 +1015,6 @@ Init_prism(void) {
     rb_define_singleton_method(rb_cPrismDebug, "named_captures", named_captures, 1);
     rb_define_singleton_method(rb_cPrismDebug, "memsize", memsize, 1);
     rb_define_singleton_method(rb_cPrismDebug, "profile_file", profile_file, 1);
-    rb_define_singleton_method(rb_cPrismDebug, "parse_serialize_file_metadata", parse_serialize_file_metadata, 2);
     rb_define_singleton_method(rb_cPrismDebug, "inspect_node", inspect_node, 1);
 
     // Next, initialize the other APIs.

include/prism.h

@@ -123,40 +123,25 @@ void pm_serialize_content(pm_parser_t *parser, pm_node_t *node, pm_buffer_t *buf
  */
 PRISM_EXPORTED_FUNCTION void pm_serialize(pm_parser_t *parser, pm_node_t *node, pm_buffer_t *buffer);
 
-/**
- * Process any additional metadata being passed into a call to the parser via
- * the pm_parse_serialize function. Since the source of these calls will be from
- * Ruby implementation internals we assume it is from a trusted source.
- *
- * Currently, this is only passing in variable scoping surrounding an eval, but
- * eventually it will be extended to hold any additional metadata.  This data
- * is serialized to reduce the calling complexity for a foreign function call
- * vs a foreign runtime making a bindable in-memory version of a C structure.
- *
- * @param parser The parser to process the metadata for.
- * @param metadata The metadata to process.
- */
-void pm_parser_metadata(pm_parser_t *parser, const char *metadata);
-
 /**
  * Parse the given source to the AST and serialize the AST to the given buffer.
  *
  * @param source The source to parse.
  * @param size The size of the source.
  * @param buffer The buffer to serialize to.
- * @param metadata The optional metadata to pass to the parser.
+ * @param data The optional data to pass to the parser.
  */
-PRISM_EXPORTED_FUNCTION void pm_parse_serialize(const uint8_t *source, size_t size, pm_buffer_t *buffer, const char *metadata);
+PRISM_EXPORTED_FUNCTION void pm_parse_serialize(const uint8_t *source, size_t size, pm_buffer_t *buffer, const char *data);
 
 /**
  * Parse and serialize the comments in the given source to the given buffer.
  *
  * @param source The source to parse.
  * @param size The size of the source.
  * @param buffer The buffer to serialize to.
- * @param metadata The optional metadata to pass to the parser.
+ * @param data The optional data to pass to the parser.
  */
-PRISM_EXPORTED_FUNCTION void pm_parse_serialize_comments(const uint8_t *source, size_t size, pm_buffer_t *buffer, const char *metadata);
+PRISM_EXPORTED_FUNCTION void pm_parse_serialize_comments(const uint8_t *source, size_t size, pm_buffer_t *buffer, const char *data);
 
 /**
  * Lex the given source and serialize to the given buffer.

include/prism/options.h

@@ -29,20 +29,20 @@ typedef struct pm_options_scope {
  */
 typedef struct {
     /** The name of the file that is currently being parsed. */
-    const char *filepath;
-
-    /**
-     * The name of the encoding that the source file is in. Note that this must
-     * correspond to a name that can be found with Encoding.find in Ruby.
-     */
-    const char *encoding;
+    pm_string_t filepath;
 
     /**
      * The line within the file that the parse starts on. This value is
      * 0-indexed.
      */
     uint32_t line;
 
+    /**
+     * The name of the encoding that the source file is in. Note that this must
+     * correspond to a name that can be found with Encoding.find in Ruby.
+     */
+    pm_string_t encoding;
+
     /**
      * The number of scopes surrounding the code that is being parsed.
      */
@@ -72,53 +72,47 @@ typedef struct {
  * @param options The options struct to set the filepath on.
  * @param filepath The filepath to set.
  */
-PRISM_EXPORTED_FUNCTION void
-pm_options_filepath_set(pm_options_t *options, const char *filepath);
+PRISM_EXPORTED_FUNCTION void pm_options_filepath_set(pm_options_t *options, const char *filepath);
 
 /**
- * Set the encoding option on the given options struct.
+ * Set the line option on the given options struct.
  *
- * @param options The options struct to set the encoding on.
- * @param encoding The encoding to set.
+ * @param options The options struct to set the line on.
+ * @param line The line to set.
  */
-PRISM_EXPORTED_FUNCTION void
-pm_options_encoding_set(pm_options_t *options, const char *encoding);
+PRISM_EXPORTED_FUNCTION void pm_options_line_set(pm_options_t *options, uint32_t line);
 
 /**
- * Set the line option on the given options struct.
+ * Set the encoding option on the given options struct.
  *
- * @param options The options struct to set the line on.
- * @param line The line to set.
+ * @param options The options struct to set the encoding on.
+ * @param encoding The encoding to set.
  */
-PRISM_EXPORTED_FUNCTION void
-pm_options_line_set(pm_options_t *options, uint32_t line);
+PRISM_EXPORTED_FUNCTION void pm_options_encoding_set(pm_options_t *options, const char *encoding);
 
 /**
  * Set the frozen string literal option on the given options struct.
  *
  * @param options The options struct to set the frozen string literal value on.
  * @param frozen_string_literal The frozen string literal value to set.
  */
-PRISM_EXPORTED_FUNCTION void
-pm_options_frozen_string_literal_set(pm_options_t *options, bool frozen_string_literal);
+PRISM_EXPORTED_FUNCTION void pm_options_frozen_string_literal_set(pm_options_t *options, bool frozen_string_literal);
 
 /**
  * Set the suppress warnings option on the given options struct.
  *
  * @param options The options struct to set the suppress warnings value on.
  * @param suppress_warnings The suppress warnings value to set.
  */
-PRISM_EXPORTED_FUNCTION void
-pm_options_suppress_warnings_set(pm_options_t *options, bool suppress_warnings);
+PRISM_EXPORTED_FUNCTION void pm_options_suppress_warnings_set(pm_options_t *options, bool suppress_warnings);
 
 /**
  * Allocate and zero out the scopes array on the given options struct.
  *
  * @param options The options struct to initialize the scopes array on.
  * @param scopes_count The number of scopes to allocate.
  */
-PRISM_EXPORTED_FUNCTION void
-pm_options_scopes_init(pm_options_t *options, size_t scopes_count);
+PRISM_EXPORTED_FUNCTION void pm_options_scopes_init(pm_options_t *options, size_t scopes_count);
 
 /**
  * Return a pointer to the scope at the given index within the given options.
@@ -127,8 +121,7 @@ pm_options_scopes_init(pm_options_t *options, size_t scopes_count);
  * @param index The index of the scope to get.
  * @return A pointer to the scope at the given index.
  */
-PRISM_EXPORTED_FUNCTION const pm_options_scope_t *
-pm_options_scope_get(const pm_options_t *options, size_t index);
+PRISM_EXPORTED_FUNCTION const pm_options_scope_t * pm_options_scope_get(const pm_options_t *options, size_t index);
 
 /**
  * Create a new options scope struct. This will hold a set of locals that are in
@@ -137,8 +130,7 @@ pm_options_scope_get(const pm_options_t *options, size_t index);
  * @param scope The scope struct to initialize.
  * @param locals_count The number of locals to allocate.
  */
-PRISM_EXPORTED_FUNCTION void
-pm_options_scope_init(pm_options_scope_t *scope, size_t locals_count);
+PRISM_EXPORTED_FUNCTION void pm_options_scope_init(pm_options_scope_t *scope, size_t locals_count);
 
 /**
  * Return a pointer to the local at the given index within the given scope.
@@ -147,15 +139,66 @@ pm_options_scope_init(pm_options_scope_t *scope, size_t locals_count);
  * @param index The index of the local to get.
  * @return A pointer to the local at the given index.
  */
-PRISM_EXPORTED_FUNCTION const pm_string_t *
-pm_options_scope_local_get(const pm_options_scope_t *scope, size_t index);
+PRISM_EXPORTED_FUNCTION const pm_string_t * pm_options_scope_local_get(const pm_options_scope_t *scope, size_t index);
 
 /**
  * Free the internal memory associated with the options.
  *
  * @param options The options struct whose internal memory should be freed.
  */
-PRISM_EXPORTED_FUNCTION void
-pm_options_free(pm_options_t *options);
+PRISM_EXPORTED_FUNCTION void pm_options_free(pm_options_t *options);
+
+/**
+ * Deserialize an options struct from the given binary string. This is used to
+ * pass options to the parser from an FFI call so that consumers of the library
+ * from an FFI perspective don't have to worry about the structure of our
+ * options structs. Since the source of these calls will be from Ruby
+ * implementation internals we assume it is from a trusted source.
+ *
+ * `data` is assumed to be a valid pointer pointing to well-formed data. The
+ * layout of this data should be the same every time, and is described below:
+ *
+ * | # bytes | field                      |
+ * | ------- | -------------------------- |
+ * | 4       | the length of the filepath |
+ * | ...     | the filepath bytes         |
+ * | 4       | the line number            |
+ * | 4       | the length the encoding    |
+ * | ...     | the encoding bytes         |
+ * | 1       | frozen string literal      |
+ * | 1       | suppress warnings          |
+ * | 4       | the number of scopes       |
+ * | ...     | the scopes                 |
+ *
+ * Each scope is layed out as follows:
+ * 
+ * | # bytes | field                      |
+ * | ------- | -------------------------- |
+ * | 4       | the number of locals       |
+ * | ...     | the locals                 |
+ *
+ * Each local is layed out as follows:
+ *
+ * | # bytes | field                      |
+ * | ------- | -------------------------- |
+ * | 4       | the length of the local    |
+ * | ...     | the local bytes            |
+ *
+ * Some additional things to note about this layout:
+ *
+ * * The filepath can have a length of 0, in which case we'll consider it an
+ *   empty string.
+ * * The line number should be 0-indexed.
+ * * The encoding can have a length of 0, in which case we'll use the default
+ *   encoding (UTF-8). If it's not 0, it should correspond to a name of an
+ *   encoding that can be passed to `Encoding.find` in Ruby.
+ * * The frozen string literal and suppress warnings fields are booleans, so
+ *   their values should be either 0 or 1.
+ * * The number of scopes can be 0.
+ *
+ * @param options The options struct to deserialize into.
+ * @param data The binary string to deserialize from.
+ */
+void pm_options_read(pm_options_t *options, const char *data);
 
 #endif

lib/prism.rb

@@ -35,13 +35,15 @@ module Prism
   private_constant :LexRipper
 
   # :call-seq:
-  #   Prism::lex_compat(source, filepath = "") -> Array
+  #   Prism::lex_compat(source, **options) -> Array
   #
   # Returns an array of tokens that closely resembles that of the Ripper lexer.
   # The only difference is that since we don't keep track of lexer state in the
   # same way, it's going to always return the NONE state.
-  def self.lex_compat(source, filepath = "")
-    LexCompat.new(source, filepath).result
+  #
+  # For supported options, see Prism::parse.
+  def self.lex_compat(source, **options)
+    LexCompat.new(source, **options).result
   end
 
   # :call-seq:

lib/prism/debug.rb

@@ -187,13 +187,5 @@ def self.prism_locals(source)
     def self.newlines(source)
       Prism.parse(source).source.offsets
     end
-
-    # :call-seq:
-    #   Debug::parse_serialize_file(filepath) -> dumped
-    #
-    # For the given file, parse the AST and dump it to a string.
-    def self.parse_serialize_file(filepath)
-      parse_serialize_file_metadata(filepath, [filepath.bytesize, filepath.b, 0].pack("LA*L"))
-    end
   end
 end