mirror of
https://github.com/openwrt/packages.git
synced 2025-09-25 17:12:51 +00:00
4410c80283
Ruby 3.4.0 is a major release that introduces several changes: - Adds `it` block parameter reference - Switches default parser to Prism - Implements Happy Eyeballs Version 2 in the socket library - Improves YJIT - Adds Modular GC - And more (see changelog for full details) Subsequent minor releases include: - 3.4.1: fixes version description - 3.4.2: routine bugfix release - 3.4.3: routine bugfix release - 3.4.4: routine bugfix release (Linux-specific) - 3.4.5: routine bugfix release, adds GCC 15 support Packaging changes: - (NEW) ruby-repl_type_completor (packaging the repl_type_completor gem) - Refreshed package dependencies - Updated `ruby_missingfiles` (detects unpacked files) to use `apk` - Refactored `ruby_find_pkgsdeps` (detects inter-package dependencies) to use the Ruby parser (Prism) instead of heuristic string matching Changelog: https://www.ruby-lang.org/en/news/2024/12/25/ruby-3-4-0-released/ Signed-off-by: Luiz Angelo Daros de Luca <luizluca@gmail.com>
492 lines
15 KiB
Ruby
492 lines
15 KiB
Ruby
#!/usr/bin/ruby -Eutf-8
|
|
# encoding: utf-8
|
|
#
|
|
# -*- mode: ruby -*-
|
|
# vi: set tabstop=8 shiftwidth=8 noexpandtab:
|
|
#
|
|
#
|
|
# Find dependencies between ruby packages
|
|
#
|
|
# Must run inside a openwrt with all *ruby* packages installed
|
|
#
|
|
#
|
|
$debug=$stderr
|
|
$info=$stderr
|
|
$error=$stderr
|
|
|
|
#$debug=File.open("/dev/null")
|
|
|
|
require "rbconfig"
|
|
RUBY_SIMPLE_VERSION = RUBY_VERSION.split(".")[0..1].join(".")
|
|
failed = false
|
|
|
|
$info.puts "Loading all installed gems (unstable after external gems are instaled/update)"
|
|
require 'rubygems'
|
|
Gem::Specification.collect{ |g| g.name.downcase }.uniq.each {|g| gem g }
|
|
|
|
$info.puts "Looking for installed ruby packages (and its files)..."
|
|
packages = []
|
|
package_files = {}
|
|
package_depends = {}
|
|
packages_json=`apk info --format json --contents --depends --match url 'http://www.ruby-lang.org/'`
|
|
require "json"
|
|
JSON.parse(packages_json).each do |pkg|
|
|
next if not pkg["contents"]
|
|
packages << pkg["name"]
|
|
package_files[pkg["name"]] = pkg["contents"].map() {|file| "/#{file}" }
|
|
package_depends[pkg["name"]] = pkg["depends"].reject{|dep| dep =~ /^lib/ or dep == "ruby" }
|
|
end
|
|
# Fake enc/utf_16 to dummy enc:
|
|
package_files["ruby-enc"]+=[RbConfig::CONFIG["rubylibdir"] + "/enc/utf_16.rb" ]
|
|
|
|
# These are Encodings that does not require extra files
|
|
builtin_enc=[
|
|
Encoding.find("ASCII-8BIT"),
|
|
Encoding.find("UTF-8"),
|
|
Encoding.find("UTF-7"),
|
|
Encoding.find("US-ASCII"),
|
|
]
|
|
|
|
# List of requires that can be simply ignored, normally because they are conditional
|
|
# requirements or to break loops
|
|
require_ignore=%w{
|
|
bundler
|
|
capistrano/version
|
|
dbm
|
|
ffi
|
|
fiber
|
|
gettext/mo
|
|
gettext/po_parser
|
|
graphviz
|
|
iconv
|
|
java
|
|
json/truffle_ruby/generator
|
|
jruby
|
|
minitest/proveit
|
|
nkf.jar
|
|
open3/jruby_windows
|
|
parser
|
|
prism/prism
|
|
profile
|
|
psych_jars
|
|
racc/cparse-jruby.jar
|
|
ruby_parser
|
|
rubygems/defaults/operating_system
|
|
rubygems/net/http
|
|
rubygems/timeout
|
|
sorted_set
|
|
stackprof
|
|
tracer
|
|
uconv
|
|
webrick
|
|
webrick/https
|
|
win32api
|
|
win32/resolv
|
|
win32/sspi
|
|
xml/encoding-ja
|
|
xmlencoding-ja
|
|
xml/parser
|
|
xmlparser
|
|
xmlscan/scanner
|
|
}
|
|
# Keep track of which of these ignores really matters
|
|
require_ignore_that_matched={}
|
|
|
|
files_ignore=%w{
|
|
extconf.rb
|
|
}
|
|
|
|
# The digestor that parsers the ruby source code for requires or use of Encodings
|
|
require "ripper"
|
|
def parse_requires_ripper(source)
|
|
requires = []
|
|
encodings = []
|
|
|
|
ast = Ripper.sexp(source)
|
|
stack = [ast]
|
|
|
|
until stack.empty?
|
|
node = stack.pop
|
|
next unless node.is_a?(Array)
|
|
|
|
case node[0]
|
|
when :command
|
|
req = nil
|
|
case node[1][1]
|
|
when "require"
|
|
#[:command, [:@ident, "require", [3, 0]], [:args_add_block, [[:string_literal, [:string_content, [:@tstring_content, "pathname", [3, 9]]]]], false]]
|
|
#[:command, [:@ident, "require", [3, 0]], [:args_add_block, [[:string_literal, [:string_content, [:@tstring_content, "rbconfig", [3, 9]]]]], false]]
|
|
#[:command, [:@ident, "require", [3, 0]], [:args_add_block, [[:string_literal, [:string_content, [:@tstring_content, "rubygems/dependency", [3, 9]]]]], false]]
|
|
# Only accepts requires with only a literal strings (without embeded expressions)
|
|
req = node[2][1][0][1][1][1] if node[2][1][0][1][1][0] == :@tstring_content and node[2][1][0][1].length == 2 rescue nil
|
|
requires << req if req
|
|
end
|
|
|
|
# node = [:command, [:@ident, "pp", [ln,col]], [:args_add_block, ...]]
|
|
if node[1][0] == :@ident && node[1][1] == "pp"
|
|
requires << "pp"
|
|
end
|
|
|
|
when :command_call
|
|
req = nil
|
|
# node = [:command_call, receiver, [:@period, ".", [ln,col]], [:@ident, "require", [ln,col]], args]
|
|
if node[3][1] == "require"
|
|
# Only accepts requires with only a literal strings (without embedded expressions)
|
|
args = node[4] rescue nil
|
|
if args && args[1] && args[1][0] && args[1][0][1] && args[1][0][1][1][0] == :@tstring_content && args[1][0][1].length == 2
|
|
req = args[1][0][1][1][1] rescue nil
|
|
requires << req if req
|
|
end
|
|
end
|
|
# The args in Encoding.find (if a string constant) should also requires the encoding (but,in practice,
|
|
# there is no case for it current ruby code
|
|
when :const_path_ref
|
|
# [:const_path_ref, [:var_ref, [:@const, "Encoding", [136, 9]]], [:@const, "US_ASCII", [136, 19]]]
|
|
if node[1][0]=:const_ref and node[1][1][1] == "Encoding" and node[2][0] = :@const
|
|
enc = node[2][1]
|
|
|
|
enc = eval("Encoding::#{enc}")
|
|
encodings << enc if enc.kind_of? Encoding
|
|
|
|
# The builtin encodings do not populate Encoding::XXX constants
|
|
requires << "enc/encdb"
|
|
end
|
|
|
|
node.each do |child|
|
|
stack << child if child.is_a?(Array)
|
|
end
|
|
|
|
when :method_add_arg
|
|
# Detects fcall :pp => [:method_add_arg, [:fcall, [:@ident, "pp", [1,0]]], ...]
|
|
if node[1][0] == :fcall && node[1][1][0] == :@ident && node[1][1][1] == "pp"
|
|
requires << "pp"
|
|
end
|
|
|
|
# Detects Kernel.pp => [:method_add_arg, [:call, [:var_ref, [:@const, "Kernel", [1,0]]], :".", [:@ident, "pp", [1,8]]], ...]
|
|
if node[1][0] == :call &&
|
|
node[1][1][0] == :var_ref && node[1][1][1][1] == "Kernel" &&
|
|
node[1][3][0] == :@ident && node[1][3][1] == "pp"
|
|
requires << "pp"
|
|
end
|
|
end
|
|
|
|
node.each do |child|
|
|
stack << child if child.is_a?(Array)
|
|
end
|
|
end
|
|
|
|
return requires, encodings
|
|
end
|
|
|
|
require "prism"
|
|
def parse_requires_prism(source)
|
|
requires = []
|
|
encodings = []
|
|
|
|
result = Prism.parse(source)
|
|
stack = [result.value] # root node
|
|
|
|
until stack.empty?
|
|
node = stack.pop
|
|
next unless node.is_a?(Prism::Node)
|
|
|
|
case node
|
|
when Prism::CallNode
|
|
# e.g. `require "foo"`
|
|
if node.name == :require && node.arguments&.arguments&.size == 1
|
|
arg = node.arguments.arguments.first
|
|
if arg.is_a?(Prism::StringNode)
|
|
requires << arg.unescaped
|
|
end
|
|
end
|
|
|
|
# Detects Kernel.pp or pp(...)
|
|
if node.name == :pp
|
|
if node.receiver.nil? # just `pp(...)`
|
|
requires << "pp"
|
|
elsif node.receiver.is_a?(Prism::ConstantReadNode) && node.receiver.name == :Kernel
|
|
requires << "pp"
|
|
end
|
|
end
|
|
|
|
when Prism::ConstantPathNode
|
|
# e.g. Encoding::US_ASCII
|
|
if node.parent.is_a?(Prism::ConstantReadNode) &&
|
|
node.parent.name == :Encoding &&
|
|
node.child.is_a?(Prism::ConstantReadNode)
|
|
|
|
enc = node.child.name.to_s
|
|
begin
|
|
enc_obj = Encoding.const_get(enc)
|
|
encodings << enc_obj if enc_obj.is_a?(Encoding)
|
|
requires << "enc/encdb"
|
|
rescue NameError
|
|
# ignore if unknown
|
|
end
|
|
end
|
|
|
|
|
|
# e.g. ::Encoding::XXX
|
|
if node.parent.is_a?(Prism::ConstantPathNode) &&
|
|
node.parent.child.is_a?(Prism::ConstantReadNode) &&
|
|
node.parent.child.name == :Encoding &&
|
|
node.child.is_a?(Prism::ConstantReadNode)
|
|
|
|
enc = node.child.name.to_s
|
|
begin
|
|
enc_obj = Encoding.const_get(enc)
|
|
encodings << enc_obj if enc_obj.is_a?(Encoding)
|
|
requires << "enc/encdb"
|
|
rescue NameError
|
|
end
|
|
end
|
|
end
|
|
|
|
# Recurse through children
|
|
node.child_nodes.compact.each do |child|
|
|
stack << child
|
|
end
|
|
end
|
|
|
|
[requires, encodings]
|
|
end
|
|
|
|
# Now check what each files in packages needs
|
|
$info.puts "Looking for requires in files..."
|
|
files_requires=Hash.new { |h,k| h[k]=[] }
|
|
packages.each do
|
|
|pkg|
|
|
$debug.puts "Checking pkg #{pkg}..."
|
|
|
|
package_files[pkg].each do
|
|
|file|
|
|
next if not File.file?(file) or not File.readable?(file)
|
|
|
|
next if files_ignore.include?(file) or files_ignore.include?(file.sub(/.*\//,""))
|
|
|
|
#file = "/usr/lib/ruby/3.4/bundler/rubygems_ext.rb"
|
|
#file = "/usr/lib/ruby/3.4/openssl/buffering.rb"
|
|
#file = "/usr/lib/ruby/3.4/unicode_normalize/normalize.rb"
|
|
#file = "/usr/lib/ruby/3.4/rdoc/encoding.rb"
|
|
#file = "/usr/lib/ruby/3.4/bundler.rb"
|
|
#file = "/usr/lib/ruby/3.4/bundler/rubygems_ext.rb"
|
|
#file = "/usr/lib/ruby/gems/3.4/gems/debug-1.11.0/lib/debug/server.rb"
|
|
|
|
f = File.open(file,"r")
|
|
line1 = f.gets()
|
|
|
|
if not file =~ /\.rb$/
|
|
next if not File.executable?(file)
|
|
next if not line1[0..1] == "#!"
|
|
next if not line1 =~ /^#!.*ruby/
|
|
$debug.puts "File #{pkg}:#{file} is a ruby script"
|
|
end
|
|
# Ignore the shebang if present
|
|
line1 = f.gets() if line1 =~ /#!.*ruby/
|
|
|
|
# Rewind to parse it all again
|
|
f.rewind()
|
|
|
|
#$debug.puts "Checking file #{pkg}:#{file}..."
|
|
requires, encodings = parse_requires_prism(f.read)
|
|
#requires2, encodings2 = parse_requires_ripper(f.read)
|
|
#if requires != requires2 or encodings != encodings2
|
|
# p pkg
|
|
# p file
|
|
# pp requires, encodings
|
|
# pp requires2, encodings2
|
|
# exit
|
|
#end
|
|
|
|
requires.reject! {|req| require_ignore_that_matched[req]=1 if require_ignore.include?(req) }
|
|
# Relative paths are always internal
|
|
requires.reject! {|req| req =~ /^\./ }
|
|
|
|
# get the magic encoding if present
|
|
if line1 =~ /^#\s*(en)?coding\s*[:=]\s*([a-zA-Z0-9_\-]+)\n/i
|
|
encodings << Encoding.find($2)
|
|
end
|
|
|
|
# ignore buildin encodings
|
|
encodings -= builtin_enc
|
|
|
|
# convert encodings to requires
|
|
requires += encodings.collect {|enc| "enc/#{enc.name.downcase.gsub("-","_")}" }
|
|
requires << "enc/encdb" if not encodings.empty?
|
|
|
|
files_requires[file] = requires
|
|
end
|
|
end
|
|
exit(1) if failed
|
|
|
|
# Check which require_ignore arr not in use
|
|
missed_ignored = (require_ignore - require_ignore_that_matched.keys).sort.join(",")
|
|
if not missed_ignored.empty?
|
|
$error.puts "These 'require_ignore' didn't match anything: ",(require_ignore - require_ignore_that_matched.keys).sort.join(","),""
|
|
end
|
|
|
|
# Add dependencies of ruby files from ruby lib.so
|
|
package_files.each do |(pkg,files)| files.each do |file|
|
|
case file
|
|
when /\/nkf\.so$/ ; files_requires[file]=files_requires[file] + ["enc/encdb"]
|
|
when /\/json\/ext\/generator\.so$/ ; files_requires[file]=files_requires[file] + ["enc/encdb"]
|
|
when /\/json\/ext\/parser\.so$/ ; files_requires[file]=files_requires[file] + ["enc/encdb"]
|
|
when /\/nkf\.so$/ ; files_requires[file]=files_requires[file] + ["enc/encdb"]
|
|
when /\/objspace\.so$/; files_requires[file]=files_requires[file] + ["tempfile"] # dump_output from ext/objspace/objspace_dump.c
|
|
when /\/openssl\.so$/; files_requires[file]=files_requires[file] + ["digest"] # Init_ossl_digest from ext/openssl/ossl_digest.c
|
|
end
|
|
end; end
|
|
|
|
$info.puts "Grouping package requirements per package"
|
|
package_requires_files = Hash.new{|h,k| h[k] = Hash.new { |h2,k2| h2[k2] = [] } }
|
|
package_files.each do |(pkg,files)|
|
|
package_requires_files[pkg]
|
|
files.each do |file|
|
|
files_requires[file].each do |requires|
|
|
package_requires_files[pkg][requires] << file
|
|
end
|
|
end
|
|
end
|
|
|
|
# For optional require or for breaking cycle dependencies
|
|
weak_dependency=Hash.new { |h,k| h[k]=[] }
|
|
weak_dependency.merge!({
|
|
"ruby-irb" =>%w{ruby-rdoc ruby-readline ruby-debug}, # irb/cmd/help.rb irb/cmd/debug.rb,3.2/irb/cmd/debug.rb
|
|
"ruby-gems" =>%w{ruby-bundler ruby-rdoc}, # rubygems.rb rubygems/server.rb rdoc/rubygems_hook
|
|
"ruby-racc" =>%w{ruby-gems}, # /usr/bin/racc*
|
|
"ruby-rake" =>%w{ruby-gems ruby-debug}, # /usr/bin/rake gems/3.3/gems/rake-13.1.0/lib/rake/application.rb
|
|
"ruby-rdoc" =>%w{ruby-readline}, # rdoc/ri/driver.rb
|
|
"ruby-testunit" =>%w{ruby-io-console}, # gems/test-unit-3.1.5/lib/test/unit/ui/console/testrunner.rb
|
|
"ruby-net-http" =>%w{ruby-open-uri} # net/http/status.rb
|
|
})
|
|
|
|
# Identify which files a package requires
|
|
$info.puts "Looking for package dependencies..."
|
|
provided_by = {}
|
|
package_provides = Hash[package_files.map() {|(pkg,files)| [pkg,files.map() {|file| file.sub(/\.(so|rb)$/,"")}]}]
|
|
package_dependencies = Hash.new { |h,k| h[k]=[] }
|
|
package_requires_files.each do
|
|
|(pkg,requires_files)|
|
|
|
|
requires_files.each do
|
|
|(require,files)|
|
|
|
|
found = provided_by[require]
|
|
if not found
|
|
# local dir or in search path are acceptables
|
|
#search_paths = (files.map() {|file| file.sub(/\/[^\/]+$/,"") } + $:).uniq
|
|
search_files = $:.map() {|path| "#{path}/#{require.sub(/\.(so|rb)$/,"")}" }
|
|
|
|
found = package_provides.detect {|(_pkg,_files)| not (_files & search_files).empty? }
|
|
|
|
if not found
|
|
$error.puts "#{pkg}: Nothing provides #{require} for #{files.collect {|file| file.sub("/usr/lib/ruby/","") }.join(",")}"
|
|
failed = true
|
|
next
|
|
end
|
|
found = found.first
|
|
provided_by[require] = found
|
|
end
|
|
|
|
if weak_dependency[pkg].include?(found)
|
|
$debug.puts "#{pkg}: #{found} provides #{require} (weak depedendency ignored) for #{files.collect {|file| file.sub("/usr/lib/ruby/","") }.join(",")}"
|
|
else
|
|
$debug.puts "#{pkg}: #{found} provides #{require} for #{files.collect {|file| file.sub("/usr/lib/ruby/","") }.join(",")}"
|
|
package_dependencies[pkg] += [found]
|
|
end
|
|
end
|
|
#break if pkg =~ /ruby-bundler.*/
|
|
end
|
|
if failed
|
|
$error.puts "There is some missing requirements not mapped to files in packages."
|
|
$error.puts "Please, fix the missing files or ignore them on require_ignore var"
|
|
exit(1)
|
|
end
|
|
|
|
# Remove self dependency
|
|
package_dependencies = Hash[package_dependencies.collect {|(pkg,deps)| [pkg,package_dependencies[pkg]=deps.uniq.sort - [pkg]]}]
|
|
package_dependencies.default = []
|
|
|
|
# Add explicity dependency
|
|
package_dependencies["ruby-enc-extra"]+=["ruby-enc"]
|
|
|
|
# Expanding dependencies, including the depedencies from required packages
|
|
$info.puts "Expanding dependencies..."
|
|
begin
|
|
changed=false
|
|
package_dependencies.each do
|
|
|(pkg,deps)|
|
|
next if deps.empty?
|
|
deps.each {|dep| $info.puts "#{pkg}: #{dep} also depends on #{pkg}" if package_dependencies[dep].include?(pkg) }
|
|
deps_new = deps.collect {|dep| [dep] + package_dependencies[dep] }.inject([],:+).uniq.sort
|
|
if not deps == deps_new
|
|
$debug.puts "#{pkg}: #{deps.join(",")} (OLD)"
|
|
$debug.puts "#{pkg}: #{deps_new.join(",")} (NEW)"
|
|
package_dependencies[pkg]=deps_new
|
|
|
|
if deps_new.include?(pkg)
|
|
$error.puts "#{pkg}: Circular dependency detected (#1)!"
|
|
exit 1
|
|
end
|
|
changed=true
|
|
end
|
|
end
|
|
end if not changed
|
|
|
|
$info.puts "Removing redundant dependencies..."
|
|
package_dependencies.each do
|
|
|(pkg,deps)|
|
|
package_dependencies[pkg]=deps.uniq - [pkg]
|
|
end
|
|
|
|
$info.puts "Checking for mutual dependencies..."
|
|
package_dependencies.each do
|
|
|(pkg,deps)|
|
|
if deps.include? pkg
|
|
$error.puts "#{pkg}: Circular dependency detected (#2)!"
|
|
failed = true
|
|
end
|
|
end
|
|
exit(1) if failed
|
|
|
|
|
|
package_dependencies2=package_dependencies.dup
|
|
package_dependencies.each do
|
|
|(pkg,deps)|
|
|
|
|
# Ignore dependencies that are already required by another dependency
|
|
deps_clean = deps.reject {|dep_suspect| deps.detect {|dep_provider|
|
|
if package_dependencies[dep_provider].include?(dep_suspect)
|
|
$info.puts "#{pkg}: #{dep_suspect} is already required by #{dep_provider}"
|
|
true
|
|
end
|
|
}}
|
|
|
|
if not deps==deps_clean
|
|
puts "before: #{deps.join(",")}"
|
|
puts "after: #{deps_clean.join(",")}"
|
|
package_dependencies2[pkg]=deps_clean
|
|
end
|
|
end
|
|
package_dependencies=package_dependencies2
|
|
|
|
$info.puts "Checking current packages dependencies..."
|
|
ok=true
|
|
package_dependencies.each do
|
|
|(pkg,deps)|
|
|
extra_dep = package_depends[pkg] - deps
|
|
$info.puts "Package #{pkg} does not need to depend on #{extra_dep.join(" ")} " if not extra_dep.empty?
|
|
missing_dep = deps - package_depends[pkg]
|
|
$info.puts "Package #{pkg} needs to depend on #{missing_dep.join(" ")} " if not missing_dep.empty?
|
|
|
|
if not extra_dep.empty? or not missing_dep.empty?
|
|
puts "define Package/#{pkg}"
|
|
puts " DEPENDS:=ruby#{([""] +deps).join(" +")}"
|
|
ok=false
|
|
end
|
|
end
|
|
|
|
puts "All dependencies are OK." if ok
|
|
|
|
__END__
|