Support lexing with Prism

Author: schneems

.github/workflows/ci.yml

@@ -43,3 +43,22 @@ jobs:
       - name: test
         run: bin/rake test
         continue-on-error: ${{ matrix.ruby == 'head' }}
+
+  test-disable-prism:
+    needs: ruby-versions
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        ruby: ${{ fromJson(needs.ruby-versions.outputs.versions) }}
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4.1.1
+      - name: Set up Ruby
+        uses: ruby/setup-ruby@v1
+        with:
+          ruby-version: ${{ matrix.ruby }}
+          bundler-cache: true
+      - name: test
+        run: SYNTAX_SUGGEST_DISABLE_PRISM=1 bin/rake test
+        continue-on-error: ${{ matrix.ruby == 'head' }}

CHANGELOG.md

@@ -1,5 +1,6 @@
 ## HEAD (unreleased)
 
+- Support prism parser (https://github.com/ruby/syntax_suggest/pull/208).
 - No longer supports EOL versions of Ruby. (https://github.com/ruby/syntax_suggest/pull/210)
 - Handle Ruby 3.3 new eval source location format (https://github.com/ruby/syntax_suggest/pull/200).
 

lib/syntax_suggest/api.rb

@@ -5,23 +5,27 @@
 require "tmpdir"
 require "stringio"
 require "pathname"
+require "timeout"
 
-# rubocop:disable Style/IdenticalConditionalBranches
-if ENV["SYNTAX_SUGGEST_DISABLE_PRISM"] # For testing dual ripper/prism support
-  require "ripper"
+# We need Ripper loaded for `Prism.lex_compat` even if we're using Prism
+# for lexing and parsing
+require "ripper"
+
+# Prism is the new parser, replacing Ripper
+#
+# We need to "dual boot" both for now because syntax_suggest
+# supports older rubies that do not ship with syntax suggest.
+#
+# We also need the ability to control loading of this library
+# so we can test that both modes work correctly in CI.
+if (value = ENV["SYNTAX_SUGGEST_DISABLE_PRISM"])
+  warn "Skipping loading prism due to SYNTAX_SUGGEST_DISABLE_PRISM=#{value}"
 else
-  # TODO remove require
-  # Allow both to be loaded to enable more atomic commits
-  require "ripper"
   begin
     require "prism"
   rescue LoadError
-    require "ripper"
   end
 end
-# rubocop:enable Style/IdenticalConditionalBranches
-
-require "timeout"
 
 module SyntaxSuggest
   # Used to indicate a default value that cannot

lib/syntax_suggest/code_line.rb

@@ -180,12 +180,19 @@ def ignore_newline_not_beg?
     #     EOM
     #     expect(lines.first.trailing_slash?).to eq(true)
     #
-    def trailing_slash?
-      last = @lex.last
-      return false unless last
-      return false unless last.type == :on_sp
+    if SyntaxSuggest.use_prism_parser?
+      def trailing_slash?
+        last = @lex.last
+        last&.type == :on_tstring_end
+      end
+    else
+      def trailing_slash?
+        last = @lex.last
+        return false unless last
+        return false unless last.type == :on_sp
 
-      last.token == TRAILING_SLASH
+        last.token == TRAILING_SLASH
+      end
     end
 
     # Endless method detection

lib/syntax_suggest/lex_all.rb

@@ -32,18 +32,15 @@ def initialize(source:, source_lines: nil)
       }
     end
 
-    # rubocop:disable Style/IdenticalConditionalBranches
     if SyntaxSuggest.use_prism_parser?
       def self.lex(source, line_number)
-        # Prism.lex_compat(source, line: line_number).value.sort_by {|values| values[0] }
-        Ripper::Lexer.new(source, "-", line_number).parse.sort_by(&:pos)
+        Prism.lex_compat(source, line: line_number).value.sort_by { |values| values[0] }
       end
     else
       def self.lex(source, line_number)
         Ripper::Lexer.new(source, "-", line_number).parse.sort_by(&:pos)
       end
     end
-    # rubocop:enable Style/IdenticalConditionalBranches
 
     def to_a
       @lex

spec/unit/api_spec.rb

@@ -8,6 +8,12 @@
 
 module SyntaxSuggest
   RSpec.describe "Top level SyntaxSuggest api" do
+    it "doesn't load prism if env var is set" do
+      skip("SYNTAX_SUGGEST_DISABLE_PRISM not set") unless ENV["SYNTAX_SUGGEST_DISABLE_PRISM"]
+
+      expect(SyntaxSuggest.use_prism_parser?).to be_falsey
+    end
+
     it "has a `handle_error` interface" do
       fake_error = Object.new
       def fake_error.message