From 2ea9336b6855b44126a55106a81e96ea68eaf8ff Mon Sep 17 00:00:00 2001 From: David Roetzel Date: Wed, 10 Jul 2024 16:25:39 +0200 Subject: [PATCH] Do not pass unknown encoding names to nokogiri. (#30987) --- app/lib/link_details_extractor.rb | 9 ++++++++- .../alternative_utf8_spelling_in_header.txt | 18 ++++++++++++++++++ spec/services/fetch_link_card_service_spec.rb | 9 +++++++++ 3 files changed, 35 insertions(+), 1 deletion(-) create mode 100644 spec/fixtures/requests/alternative_utf8_spelling_in_header.txt diff --git a/app/lib/link_details_extractor.rb b/app/lib/link_details_extractor.rb index 72992d2f42..d81f4a3062 100644 --- a/app/lib/link_details_extractor.rb +++ b/app/lib/link_details_extractor.rb @@ -274,7 +274,7 @@ class LinkDetailsExtractor end def detect_encoding_and_parse_document - [detect_encoding, nil, @html_charset].uniq.each do |encoding| + [detect_encoding, nil, header_encoding].uniq.each do |encoding| document = Nokogiri::HTML(@html, nil, encoding) return document if document.to_s.valid_encoding? end @@ -286,6 +286,13 @@ class LinkDetailsExtractor guess&.fetch(:confidence, 0).to_i > 60 ? guess&.fetch(:encoding, nil) : nil end + def header_encoding + Encoding.find(@html_charset).name if @html_charset + rescue ArgumentError + # Encoding from HTTP header is not recognized by ruby + nil + end + def detector @detector ||= CharlockHolmes::EncodingDetector.new.tap do |detector| detector.strip_tags = true diff --git a/spec/fixtures/requests/alternative_utf8_spelling_in_header.txt b/spec/fixtures/requests/alternative_utf8_spelling_in_header.txt new file mode 100644 index 0000000000..7aaea370e8 --- /dev/null +++ b/spec/fixtures/requests/alternative_utf8_spelling_in_header.txt @@ -0,0 +1,18 @@ +HTTP/1.1 200 OK +server: nginx +date: Thu, 13 Jun 2024 14:33:13 GMT +content-type: text/html; charset=utf8 +content-length: 192 +accept-ranges: bytes + + + + + + Webserver Configs R Us + + +

Welcome

+

Sneaky non-UTF character: á

+ + diff --git a/spec/services/fetch_link_card_service_spec.rb b/spec/services/fetch_link_card_service_spec.rb index 342902cdb3..2f64f40558 100644 --- a/spec/services/fetch_link_card_service_spec.rb +++ b/spec/services/fetch_link_card_service_spec.rb @@ -32,6 +32,7 @@ RSpec.describe FetchLinkCardService do stub_request(:get, 'http://example.com/aergerliche-umlaute').to_return(request_fixture('redirect_with_utf8_url.txt')) stub_request(:get, 'http://example.com/page_without_title').to_return(request_fixture('page_without_title.txt')) stub_request(:get, 'http://example.com/long_canonical_url').to_return(request_fixture('long_canonical_url.txt')) + stub_request(:get, 'http://example.com/alternative_utf8_spelling_in_header').to_return(request_fixture('alternative_utf8_spelling_in_header.txt')) Rails.cache.write('oembed_endpoint:example.com', oembed_cache) if oembed_cache @@ -292,6 +293,14 @@ RSpec.describe FetchLinkCardService do expect(status.preview_card).to be_nil end end + + context 'with a URL where the `Content-Type` header uses `utf8` instead of `utf-8`' do + let(:status) { Fabricate(:status, text: 'test http://example.com/alternative_utf8_spelling_in_header') } + + it 'does not create a preview card' do + expect(status.preview_card.title).to eq 'Webserver Configs R Us' + end + end end context 'with a remote status' do