Add FFI parser backend foundation for JRuby support

.github/workflows/ruby.yml

@@ -83,6 +83,37 @@ jobs:
       - name: Run test
         run: |
           bundle exec rake ${{ matrix.job }}
+  ffi_backend:
+    runs-on: "ubuntu-latest"
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          # The FFI backend on MRI, forced with RBS_FFI_BACKEND=1. This is
+          # the cheap lane that catches most FFI backend regressions.
+          - ruby: "4.0"
+            rbs_ffi_backend: "1"
+          # The FFI backend on JRuby, where it is the only backend.
+          - ruby: jruby
+            rbs_ffi_backend: ""
+    env:
+      RBS_FFI_BACKEND: ${{ matrix.rbs_ffi_backend }}
+      # The main Gemfile pulls in development gems whose C extensions cannot
+      # be built on JRuby; this lane only needs the parser test subset.
+      BUNDLE_GEMFILE: gemfiles/ffi_backend.gemfile
+    steps:
+      - uses: actions/checkout@v6
+      - uses: ruby/setup-ruby@v1
+        with:
+          ruby-version: ${{ matrix.ruby }}
+          bundler: none
+      - name: Set working directory as safe
+        run: git config --global --add safe.directory $(pwd)
+      - name: Install dependencies
+        run: bundle install --jobs 4 --retry 3
+      - name: Run parser tests
+        run: bundle exec rake test:parser
+
   C99_compile:
     runs-on: macos-latest
     strategy:

.gitignore

@@ -18,6 +18,7 @@
 lib/**/*.bundle
 lib/**/*.so
 lib/**/*.dll
+lib/**/*.dylib
 doc/
 
 **/*.gem
@@ -29,3 +30,4 @@ ext/rbs_extension/.cache
 # Rust crate vendored RBS source (managed by rake rust:rbs:sync or rust:rbs:symlink)
 rust/ruby-rbs-sys/vendor/rbs/
 rust/ruby-rbs/vendor/rbs/
+gemfiles/*.gemfile.lock

Gemfile

@@ -23,6 +23,9 @@ gem "activesupport", "~> 7.0"
 gem "extconf_compile_commands_json"
 gem "irb"
 
+# FFI parser backend (non-MRI implementations, or RBS_FFI_BACKEND=1 on MRI)
+gem "ffi", require: false
+
 group :libs do
   # Libraries required for stdlib test
   gem "abbrev"

Gemfile.lock

@@ -204,6 +204,7 @@ DEPENDENCIES
   dbm
   digest
   extconf_compile_commands_json
+  ffi
   fileutils
   goodcheck
   irb

Rakefile

@@ -37,6 +37,37 @@ Rake::TestTask.new(test: :compile, &test_config)
 
 multitask :default => [:test, :stdlib_test, :typecheck_test, :rubocop, :validate, :test_doc]
 
+namespace :compile do
+  desc "Build the core parser as a shared library (librbs) for the FFI backend"
+  task :librbs do
+    require "tmpdir"
+    extconf = File.join(__dir__, "ext/rbs_extension/extconf.rb")
+    Dir.mktmpdir do |dir|
+      Dir.chdir(dir) do
+        sh({ "RBS_FFI_BACKEND" => "1" }, ruby, extconf)
+      end
+    end
+  end
+end
+
+# The parser-focused test subset, used as the acceptance suite for the FFI
+# backend (RBS_FFI_BACKEND=1 on MRI, or non-MRI implementations).
+Rake::TestTask.new(:"test:parser") do |t|
+  t.libs << "test"
+  t.libs << "lib"
+  t.test_files = FileList[
+    "test/rbs/buffer_test.rb",
+    "test/rbs/location_test.rb",
+    "test/rbs/parser_test.rb",
+    "test/rbs/type_parsing_test.rb",
+    "test/rbs/method_type_parsing_test.rb",
+    "test/rbs/signature_parsing_test.rb",
+    "test/rbs/inline_annotation_parsing_test.rb",
+    "test/rbs/inline_parser_test.rb",
+  ]
+end
+task :"test:parser" => (RUBY_ENGINE != "ruby" || !ENV["RBS_FFI_BACKEND"].to_s.empty? ? :"compile:librbs" : :compile)
+
 task :lexer do
   sh "re2c -W --no-generation-date -o src/lexer.c src/lexer.re"
   sh "clang-format -i -style=file src/lexer.c"
@@ -48,8 +79,8 @@ task :confirm_lexer => :lexer do
 end
 
 task :confirm_templates => :templates do
-  puts "Testing if generated code under include and src is updated with respect to templates"
-  sh "git diff --exit-code -- include src"
+  puts "Testing if generated code under include, src and lib is updated with respect to templates"
+  sh "git diff --exit-code -- include src lib/rbs/parser/deserializer.rb"
 end
 
 # Task to format C code using clang-format
@@ -160,6 +191,9 @@ task :templates do
   sh "#{ruby} templates/template.rb include/rbs/ast.h"
   sh "#{ruby} templates/template.rb src/ast.c"
 
+  sh "#{ruby} templates/template.rb src/serializer.c"
+  sh "#{ruby} templates/template.rb lib/rbs/parser/deserializer.rb"
+
   # Format the generated files
   Rake::Task["format:c"].invoke
 end

Steepfile

@@ -6,6 +6,13 @@ target :lib do
   ignore(
     "lib/rbs/test",
     # "lib/rbs/test.rb"
+
+    # The FFI backend depends on the ffi gem, which has no RBS signatures,
+    # and the deserializer is generated code. The pure-Ruby Location mirrors
+    # the C extension implementation, which is not type checked either.
+    "lib/rbs/parser/ffi.rb",
+    "lib/rbs/parser/deserializer.rb",
+    "lib/rbs/location.rb",
   )
 
   library "pathname", "json", "logger", "monitor", "tsort", "uri", 'dbm', 'pstore', 'singleton', 'shellwords', 'fileutils', 'find', 'digest', 'prettyprint', 'yaml', "psych", "securerandom"

ext/rbs_extension/extconf.rb

@@ -1,40 +1,88 @@
-require 'mkmf'
+if RUBY_ENGINE == "ruby" && ENV["RBS_FFI_BACKEND"].to_s.empty?
+  require 'mkmf'
 
-$INCFLAGS << " -I$(top_srcdir)" if $extmk
-$INCFLAGS << " -I$(srcdir)/../../include"
+  $INCFLAGS << " -I$(top_srcdir)" if $extmk
+  $INCFLAGS << " -I$(srcdir)/../../include"
 
-$VPATH << "$(srcdir)/../../src"
-$VPATH << "$(srcdir)/../../src/util"
-$VPATH << "$(srcdir)/ext/rbs_extension"
+  $VPATH << "$(srcdir)/../../src"
+  $VPATH << "$(srcdir)/../../src/util"
+  $VPATH << "$(srcdir)/ext/rbs_extension"
 
-root_dir = File.expand_path('../../../', __FILE__)
-$srcs = Dir.glob("#{root_dir}/src/**/*.c") +
-        Dir.glob("#{root_dir}/ext/rbs_extension/*.c")
+  root_dir = File.expand_path('../../../', __FILE__)
+  $srcs = Dir.glob("#{root_dir}/src/**/*.c") +
+          Dir.glob("#{root_dir}/ext/rbs_extension/*.c")
 
-append_cflags [
-  '-std=gnu99',
-  '-Wimplicit-fallthrough',
-  '-Wunused-result',
-  '-Wc++-compat',
-  '-Wnullable-to-nonnull-conversion',
-]
+  append_cflags [
+    '-std=gnu99',
+    '-Wimplicit-fallthrough',
+    '-Wunused-result',
+    '-Wc++-compat',
+    '-Wnullable-to-nonnull-conversion',
+  ]
 
-if ENV['DEBUG']
-  append_cflags ['-O0', '-pg']
+  if ENV['DEBUG']
+    append_cflags ['-O0', '-pg']
+  else
+    append_cflags ['-DNDEBUG']
+  end
+  if ENV["TEST_NO_C23"]
+    puts "Adding -Wc2x-extensions to CFLAGS"
+    $CFLAGS << " -Werror -Wc2x-extensions"
+  end
+
+  create_makefile 'rbs_extension'
+
+  # Only generate compile_commands.json when compiling through Rake tasks
+  # This is to avoid adding extconf_compile_commands_json as a runtime dependency
+  if ENV["COMPILE_COMMANDS_JSON"]
+    require 'extconf_compile_commands_json'
+    ExtconfCompileCommandsJson.generate!
+    ExtconfCompileCommandsJson.symlink!
+  end
 else
-  append_cflags ['-DNDEBUG']
-end
-if ENV["TEST_NO_C23"]
-  puts "Adding -Wc2x-extensions to CFLAGS"
-  $CFLAGS << " -Werror -Wc2x-extensions"
-end
+  # Non-MRI implementations (JRuby, TruffleRuby) cannot load MRI C extensions.
+  # Instead, build the Ruby-independent core parser (src/ only, no
+  # ext/rbs_extension/ sources) as a plain shared library, loaded at runtime
+  # through the ffi gem by lib/rbs/parser/ffi.rb.
+  #
+  # Setting RBS_FFI_BACKEND=1 forces this path on MRI, which is how the FFI
+  # backend is developed and tested without a JRuby installation.
+  require 'rbconfig'
+
+  root_dir = File.expand_path('../../../', __FILE__)
+
+  soext = RbConfig::CONFIG["SOEXT"] ||
+          (RbConfig::CONFIG["host_os"] =~ /darwin/ ? "dylib" : "so")
+  cc = RbConfig::CONFIG["CC"] || ENV["CC"] || "cc"
+  output = File.join(root_dir, "lib", "rbs", "librbs.#{soext}")
+
+  sources = Dir.glob("#{root_dir}/src/**/*.c")
+
+  command = [
+    cc,
+    "-O2",
+    "-fPIC",
+    "-std=gnu99",
+    "-fvisibility=default",
+    "-DNDEBUG",
+    "-I#{root_dir}/include",
+    "-shared",
+    "-o", output,
+    *sources,
+  ]
 
-create_makefile 'rbs_extension'
+  puts "Building librbs for the FFI backend: #{output}"
+  puts command.join(" ")
+  system(*command) or raise "Failed to build librbs with: #{command.join(" ")}"
 
-# Only generate compile_commands.json when compiling through Rake tasks
-# This is to avoid adding extconf_compile_commands_json as a runtime dependency
-if ENV["COMPILE_COMMANDS_JSON"]
-  require 'extconf_compile_commands_json'
-  ExtconfCompileCommandsJson.generate!
-  ExtconfCompileCommandsJson.symlink!
+  # RubyGems expects extconf.rb to produce a Makefile; librbs is already
+  # built at this point, so all targets are no-ops.
+  File.write("Makefile", <<~MAKEFILE)
+    all:
+    \t@true
+    install:
+    \t@true
+    clean:
+    \t@true
+  MAKEFILE
 end

gemfiles/ffi_backend.gemfile

@@ -0,0 +1,19 @@
+# Minimal Gemfile for the FFI parser backend CI lane (`rake test:parser` on
+# JRuby, or on MRI with RBS_FFI_BACKEND=1).
+#
+# The main Gemfile pulls in development gems whose C extensions cannot be
+# built on JRuby (e.g. zlib via rubocop-on-rbs, bigdecimal via
+# activesupport), so this lane installs only what the parser test subset
+# needs.
+
+source "https://rubygems.org"
+
+gemspec path: "../"
+
+gem "rake"
+gem "rake-compiler" # Required by the Rakefile (rake/extensiontask)
+gem "test-unit"
+
+# Used by the backend when forced on MRI with RBS_FFI_BACKEND=1; JRuby and
+# TruffleRuby bundle ffi as a default gem.
+gem "ffi", require: false

include/rbs/serializer.h

@@ -0,0 +1,40 @@
+#ifndef RBS__SERIALIZER_H
+#define RBS__SERIALIZER_H
+
+#include "rbs/ast.h"
+#include "rbs/util/rbs_buffer.h"
+#include "rbs/util/rbs_constant_pool.h"
+
+/**
+ * Serializes AST nodes into a compact binary format, consumed by the
+ * pure-Ruby deserializer of the FFI parser backend
+ * (lib/rbs/parser/deserializer.rb).
+ *
+ * The format is private to RBS: it is produced and consumed by the same gem
+ * version, with both sides generated from config.yml, and carries no
+ * versioning or compatibility guarantees. All integers are little-endian,
+ * independent of the host platform.
+ */
+typedef struct {
+    rbs_allocator_t *allocator;
+    rbs_buffer_t *buffer;
+    const rbs_constant_pool_t *constant_pool;
+} rbs_serializer_t;
+
+void rbs_serializer_write_u8(rbs_serializer_t *serializer, uint8_t value);
+void rbs_serializer_write_u32(rbs_serializer_t *serializer, uint32_t value);
+void rbs_serializer_write_i32(rbs_serializer_t *serializer, int32_t value);
+void rbs_serializer_write_string(rbs_serializer_t *serializer, rbs_string_t string);
+
+/**
+ * Writes the serialized representation of the given node (or 0 for NULL)
+ * into the serializer's buffer.
+ */
+void rbs_serializer_write_node(rbs_serializer_t *serializer, const rbs_node_t *node);
+
+/**
+ * Writes a node list: u32 element count followed by each node.
+ */
+void rbs_serializer_write_node_list(rbs_serializer_t *serializer, rbs_node_list_t *list);
+
+#endif

lib/rbs.rb

@@ -69,7 +69,14 @@
 require "rbs/type_alias_regularity"
 require "rbs/collection"
 
-require "rbs_extension"
+if RUBY_ENGINE == "ruby" && ENV["RBS_FFI_BACKEND"].to_s.empty?
+  require "rbs_extension"
+else
+  # Non-MRI implementations cannot load the C extension. Load the FFI-based
+  # parser backend and the pure-Ruby RBS::Location instead.
+  require "rbs/location"
+  require "rbs/parser/ffi"
+end
 require "rbs/parser_aux"
 require "rbs/location_aux"