API tools faq
paste
Guest User

Untitled

a guest
Jul 30th, 2018
223
0
Never
Add comment
Not a member of Pastebin yet? Sign Up, it unlocks many cool features!
text 17.30 KB | None | 0 0
raw download clone embed print report
  1. From 6f0ed5bf7d719f23e900cf25e56a5eba509750c4 Mon Sep 17 00:00:00 2001
  2. From: Cezary Baginski <cezary.baginski@gmail.com>
  3. Date: Tue, 27 Apr 2010 16:41:46 +0200
  4. Subject: [PATCH 1/3] activesupport: added external_encode! to string
  5.  
  6. ---
  7. .../active_support/core_ext/string/multibyte.rb | 42 ++++++++++-
  8. activesupport/test/core_ext/string_ext_test.rb | 83 ++++++++++++++++++++
  9. 2 files changed, 123 insertions(+), 2 deletions(-)
  10.  
  11. diff --git a/activesupport/lib/active_support/core_ext/string/multibyte.rb b/activesupport/lib/active_support/core_ext/string/multibyte.rb
  12. index 42e053d..b591c3d 100644
  13. --- a/activesupport/lib/active_support/core_ext/string/multibyte.rb
  14. +++ b/activesupport/lib/active_support/core_ext/string/multibyte.rb
  15. @@ -43,17 +43,21 @@ class String
  16. self
  17. end
  18. end
  19. -
  20. +
  21. # Returns true if the string has UTF-8 semantics (a String used for purely byte resources is unlikely to have
  22. # them), returns false otherwise.
  23. def is_utf8?
  24. ActiveSupport::Multibyte::Chars.consumes?(self)
  25. end
  26. +
  27. + def external_encode!(source_encoding = nil)
  28. + self
  29. + end
  30. else
  31. def mb_chars #:nodoc
  32. self
  33. end
  34. -
  35. +
  36. def is_utf8? #:nodoc
  37. case encoding
  38. when Encoding::UTF_8
  39. @@ -64,5 +68,39 @@ class String
  40. false
  41. end
  42. end
  43. +
  44. + def external_encode!(source_encoding = nil)
  45. +
  46. + unless self.encoding == Encoding::ASCII_8BIT
  47. + if source_encoding
  48. + unless source_encoding == self.encoding
  49. + raise ArgumentError, "source_encoding given (#{source_encoding}) for non-binary string (#{self.encoding})"
  50. + end
  51. + end
  52. + end
  53. +
  54. + enc = Encoding.default_external
  55. +
  56. + # pick anything but binary if possible
  57. + source_encoding ||= (self.encoding == Encoding::ASCII_8BIT ? enc : self.encoding )
  58. +
  59. + # noop case
  60. + return self if source_encoding == enc and enc == self.encoding
  61. +
  62. + # allow force_encoding to work on frozen strings
  63. + s = (self.frozen? ? self.dup : self)
  64. +
  65. + # correct the string's encoding if necessary
  66. + s = s.force_encoding(source_encoding) if s.encoding == Encoding::ASCII_8BIT
  67. +
  68. + # Two things happen here:
  69. + # - characters are validated (unless source is utf-8 already)
  70. + # - non utf-8 strings are converted to utf-8 before converted to binary
  71. + s = s.encode!('utf-8')
  72. +
  73. + # convert to the default
  74. + return s.force_encoding(Encoding::ASCII_8BIT) if enc == Encoding::ASCII_8BIT
  75. + s.encode!(enc)
  76. + end
  77. end
  78. end
  79. diff --git a/activesupport/test/core_ext/string_ext_test.rb b/activesupport/test/core_ext/string_ext_test.rb
  80. index 97b08da..957ed94 100644
  81. --- a/activesupport/test/core_ext/string_ext_test.rb
  82. +++ b/activesupport/test/core_ext/string_ext_test.rb
  83. @@ -252,6 +252,89 @@ class CoreExtStringMultibyteTest < ActiveSupport::TestCase
  84. assert UNICODE_STRING.mb_chars.kind_of?(String)
  85. end
  86. end
  87. +
  88. + if RUBY_VERSION >= '1.9'
  89. + def with_external_encoding(encoding)
  90. + old_external = Encoding.default_external
  91. + silence_warnings { Encoding.default_external = encoding }
  92. + yield
  93. + silence_warnings { Encoding.default_external = old_external}
  94. + end
  95. +
  96. + def reencode(str)
  97. + enc = Encoding::default_external
  98. + if enc == Encoding::ASCII_8BIT
  99. + str.dup.encode!('utf-8').dup.force_encoding(enc)
  100. + else
  101. + str.dup.encode!(enc)
  102. + end
  103. + end
  104. +
  105. + def assert_equal_encodings(expected, str)
  106. + assert_equal [Encoding::default_external, expected.dup.force_encoding(Encoding::ASCII_8BIT) ],
  107. + [str.encoding, str.dup.force_encoding(Encoding::ASCII_8BIT)]
  108. + end
  109. +
  110. + UTF_STRING = '日本語'.freeze
  111. + EUC_STRING = UTF_STRING.dup.encode!('euc-jp').freeze
  112. + SJIS_STRING = UTF_STRING.dup.encode!('sjis').freeze
  113. + EUC_STRING_AS_BINARY = EUC_STRING.dup.force_encoding('ascii-8bit').freeze
  114. +
  115. + def test_external_encode_assumes_current_encoding_for_binary_input
  116. + with_external_encoding('euc-jp') do
  117. + expected = reencode(UTF_STRING)
  118. + assert_equal_encodings expected, EUC_STRING_AS_BINARY.external_encode!
  119. + end
  120. + end
  121. +
  122. + def test_encoding_fails_for_incompatible_strings
  123. + with_external_encoding('euc-jp') do
  124. + begin
  125. + SJIS_STRING.dup.force_encoding('ascii-8bit').external_encode!
  126. + flunk 'Should have raised incompatibility error'
  127. + rescue Encoding::InvalidByteSequenceError => error
  128. + assert_match %r!.*on\sEUC-JP!, error.message
  129. + end
  130. + begin
  131. + SJIS_STRING.dup.force_encoding('ascii-8bit').external_encode!('euc-jp')
  132. + flunk 'Should have raised incompatibility error'
  133. + rescue Encoding::InvalidByteSequenceError => error
  134. + assert_match %r!.*on\sEUC-JP!, error.message
  135. + end
  136. + end
  137. + end
  138. +
  139. + def test_external_encode_assumes_given_encoding_for_binary_input
  140. + with_external_encoding('ascii-8bit') do
  141. + expected = reencode(UTF_STRING)
  142. + assert_equal_encodings expected, EUC_STRING_AS_BINARY.external_encode!(Encoding::EUC_JP)
  143. + end
  144. + end
  145. +
  146. + def test_external_encoding_converts_to_unicode_before_binary
  147. + with_external_encoding('ascii-8bit') do
  148. + expected = reencode(UTF_STRING)
  149. + assert_equal_encodings expected, EUC_STRING.external_encode!
  150. + end
  151. + end
  152. +
  153. + def test_external_encode_conversions
  154. + %w(euc-jp sjis ascii-8bit utf-8).each do |encoding|
  155. + with_external_encoding(encoding) do
  156. + expected = reencode(UTF_STRING)
  157. + assert_equal_encodings expected, UTF_STRING.external_encode!
  158. + assert_equal_encodings expected, EUC_STRING.external_encode!
  159. + assert_equal_encodings expected, EUC_STRING_AS_BINARY.external_encode!(Encoding::EUC_JP)
  160. + end
  161. + end
  162. + end
  163. + else
  164. + def test_external_encode_does_nothing
  165. + with_kcode('UTF8') do
  166. + assert_equal '日本語', '日本語'.external_encode!
  167. + end
  168. + end
  169. + end
  170. end
  171.  
  172. =begin
  173. --
  174. 1.6.3.3
  175.  
  176.  
  177. From dc4f6f2801b593da468a08c64ec1ba83de8216c5 Mon Sep 17 00:00:00 2001
  178. From: Cezary Baginski <cezary.baginski@gmail.com>
  179. Date: Tue, 27 Apr 2010 16:43:22 +0200
  180. Subject: [PATCH 2/3] actionpack: added render error line number fixtures
  181.  
  182. ---
  183. .../fixtures/test/_raise_at_2_no_newline.html.erb | 2 ++
  184. actionpack/test/fixtures/test/_raise_at_3.html.erb | 3 +++
  185. 2 files changed, 5 insertions(+), 0 deletions(-)
  186. create mode 100644 actionpack/test/fixtures/test/_raise_at_2_no_newline.html.erb
  187. create mode 100644 actionpack/test/fixtures/test/_raise_at_3.html.erb
  188.  
  189. diff --git a/actionpack/test/fixtures/test/_raise_at_2_no_newline.html.erb b/actionpack/test/fixtures/test/_raise_at_2_no_newline.html.erb
  190. new file mode 100644
  191. index 0000000..22404d4
  192. --- /dev/null
  193. +++ b/actionpack/test/fixtures/test/_raise_at_2_no_newline.html.erb
  194. @@ -0,0 +1,2 @@
  195. +<%# encoding: us-ascii %><%= "line1" %>
  196. +<%= error + "line2" %>
  197. diff --git a/actionpack/test/fixtures/test/_raise_at_3.html.erb b/actionpack/test/fixtures/test/_raise_at_3.html.erb
  198. new file mode 100644
  199. index 0000000..3ac0ec3
  200. --- /dev/null
  201. +++ b/actionpack/test/fixtures/test/_raise_at_3.html.erb
  202. @@ -0,0 +1,3 @@
  203. +<%# encoding: us-ascii %>
  204. +<%= "line2" %>
  205. +<%= error + "line3" %>
  206. --
  207. 1.6.3.3
  208.  
  209.  
  210. From 7681266317bd21dca51ec90c0ae5bb5c2a3be6a8 Mon Sep 17 00:00:00 2001
  211. From: Cezary Baginski <cezary.baginski@gmail.com>
  212. Date: Tue, 27 Apr 2010 16:47:09 +0200
  213. Subject: [PATCH 3/3] actionpack: ERb encoding support in Ruby 1.9
  214.  
  215. ---
  216. .../lib/action_view/template/handlers/erb.rb | 25 ++++++++--
  217. .../test/fixtures/test/_euc_jp_magic.html.erb | 1 +
  218. actionpack/test/fixtures/test/_sjis_magic.html.erb | 1 +
  219. actionpack/test/fixtures/test/_utf8_magic.html.erb | 2 +
  220. .../test/fixtures/test/sjis_euc_partials.html.erb | 1 +
  221. .../fixtures/test/sjis_euc_utf_partials.html.erb | 1 +
  222. actionpack/test/fixtures/test/utf8.html.erb | 3 -
  223. actionpack/test/fixtures/test/utf8_magic.html.erb | 5 --
  224. actionpack/test/template/render_test.rb | 51 +++++++++++++++++--
  225. 9 files changed, 72 insertions(+), 18 deletions(-)
  226. create mode 100644 actionpack/test/fixtures/test/_euc_jp_magic.html.erb
  227. create mode 100644 actionpack/test/fixtures/test/_sjis_magic.html.erb
  228. create mode 100644 actionpack/test/fixtures/test/_utf8_magic.html.erb
  229. create mode 100644 actionpack/test/fixtures/test/sjis_euc_partials.html.erb
  230. create mode 100644 actionpack/test/fixtures/test/sjis_euc_utf_partials.html.erb
  231. delete mode 100644 actionpack/test/fixtures/test/utf8_magic.html.erb
  232.  
  233. diff --git a/actionpack/lib/action_view/template/handlers/erb.rb b/actionpack/lib/action_view/template/handlers/erb.rb
  234. index 705c2bf..8165069 100644
  235. --- a/actionpack/lib/action_view/template/handlers/erb.rb
  236. +++ b/actionpack/lib/action_view/template/handlers/erb.rb
  237. @@ -71,10 +71,27 @@ module ActionView
  238. self.erb_implementation = Erubis
  239.  
  240. def compile(template)
  241. - source = template.source.gsub(/\A(<%(#.*coding[:=]\s*(\S+)\s*)-?%>)\s*\n?/, '')
  242. - erb = "<% __in_erb_template=true %>#{source}"
  243. - result = self.class.erb_implementation.new(erb, :trim=>(self.class.erb_trim_mode == "-")).src
  244. - result = "#{$2}\n#{result}" if $2
  245. + if RUBY_VERSION >= '1.9'
  246. + # Convert to binary so the regexp works
  247. + source = template.source.dup.force_encoding(Encoding::ASCII_8BIT)
  248. + source = source.gsub(/\A(<%(#.*coding[:=]\s*(\S+)\s*)-?%>)[ \t\r\f]*(\n?)/, ''.force_encoding(Encoding::ASCII_8BIT))
  249. +
  250. + # use the encoding from magic or assume default
  251. + detected_encoding = $3 ? Encoding.find($3) : Encoding::default_external
  252. + newline = $4
  253. +
  254. + # reencode to default, passing detected as source encoding
  255. + reencoded_source = source.external_encode!(detected_encoding)
  256. +
  257. + # render - interpolation honors the encoding
  258. + erb = "<% __in_erb_template=true %>#{newline}#{reencoded_source}"
  259. + result = self.class.erb_implementation.new(erb, :trim=>(self.class.erb_trim_mode == "-")).src
  260. + result = result.external_encode!
  261. + else
  262. + source = template.source.gsub(/\A(<%(#.*coding[:=]\s*(\S+)\s*)-?%>)[ \t\r\f]*(\n?)/, '')
  263. + erb = "<% __in_erb_template=true %>#{$4}#{source}"
  264. + result = self.class.erb_implementation.new(erb, :trim=>(self.class.erb_trim_mode == "-")).src
  265. + end
  266. result
  267. end
  268. end
  269. diff --git a/actionpack/test/fixtures/test/_euc_jp_magic.html.erb b/actionpack/test/fixtures/test/_euc_jp_magic.html.erb
  270. new file mode 100644
  271. index 0000000..ab796f4
  272. --- /dev/null
  273. +++ b/actionpack/test/fixtures/test/_euc_jp_magic.html.erb
  274. @@ -0,0 +1 @@
  275. +<%# encoding: euc-jp %>��ܸ�Υƥ�����
  276. diff --git a/actionpack/test/fixtures/test/_sjis_magic.html.erb b/actionpack/test/fixtures/test/_sjis_magic.html.erb
  277. new file mode 100644
  278. index 0000000..4e8040a
  279. --- /dev/null
  280. +++ b/actionpack/test/fixtures/test/_sjis_magic.html.erb
  281. @@ -0,0 +1 @@
  282. +<%# encoding: sjis %>���{��̃e�L�X�g
  283. diff --git a/actionpack/test/fixtures/test/_utf8_magic.html.erb b/actionpack/test/fixtures/test/_utf8_magic.html.erb
  284. new file mode 100644
  285. index 0000000..8f113c5
  286. --- /dev/null
  287. +++ b/actionpack/test/fixtures/test/_utf8_magic.html.erb
  288. @@ -0,0 +1,2 @@
  289. +<%# encoding: utf-8 -%>
  290. +Русский <%= render :partial => 'test/utf8_partial_magic' %>
  291. diff --git a/actionpack/test/fixtures/test/sjis_euc_partials.html.erb b/actionpack/test/fixtures/test/sjis_euc_partials.html.erb
  292. new file mode 100644
  293. index 0000000..c6e9f59
  294. --- /dev/null
  295. +++ b/actionpack/test/fixtures/test/sjis_euc_partials.html.erb
  296. @@ -0,0 +1 @@
  297. +<%# encoding: us-ascii %><%= render(:file => 'test/_sjis_magic.html.erb') %><%= render(:file => 'test/_euc_jp_magic.html.erb') %>
  298. diff --git a/actionpack/test/fixtures/test/sjis_euc_utf_partials.html.erb b/actionpack/test/fixtures/test/sjis_euc_utf_partials.html.erb
  299. new file mode 100644
  300. index 0000000..89faa3c
  301. --- /dev/null
  302. +++ b/actionpack/test/fixtures/test/sjis_euc_utf_partials.html.erb
  303. @@ -0,0 +1 @@
  304. +<%# encoding: euc-jp %><%= render(:file => 'test/_sjis_magic.html.erb') %><%= render(:file => 'test/_euc_jp_magic.html.erb') %><%= render(:file => 'test/_utf8_magic.html.erb') %>
  305. diff --git a/actionpack/test/fixtures/test/utf8.html.erb b/actionpack/test/fixtures/test/utf8.html.erb
  306. index ac98c2f..1487758 100644
  307. --- a/actionpack/test/fixtures/test/utf8.html.erb
  308. +++ b/actionpack/test/fixtures/test/utf8.html.erb
  309. @@ -1,4 +1 @@
  310. Русский <%= render :partial => 'test/utf8_partial' %>
  311. -<%= "日".encoding %>
  312. -<%= @output_buffer.encoding %>
  313. -<%= __ENCODING__ %>
  314. diff --git a/actionpack/test/fixtures/test/utf8_magic.html.erb b/actionpack/test/fixtures/test/utf8_magic.html.erb
  315. deleted file mode 100644
  316. index 257279c..0000000
  317. --- a/actionpack/test/fixtures/test/utf8_magic.html.erb
  318. +++ /dev/null
  319. @@ -1,5 +0,0 @@
  320. -<%# encoding: utf-8 -%>
  321. -Русский <%= render :partial => 'test/utf8_partial_magic' %>
  322. -<%= "日".encoding %>
  323. -<%= @output_buffer.encoding %>
  324. -<%= __ENCODING__ %>
  325. diff --git a/actionpack/test/template/render_test.rb b/actionpack/test/template/render_test.rb
  326. index c9a50da..22a2c64 100644
  327. --- a/actionpack/test/template/render_test.rb
  328. +++ b/actionpack/test/template/render_test.rb
  329. @@ -114,6 +114,22 @@ module RenderTestCases
  330. assert_equal File.expand_path("#{FIXTURE_LOAD_PATH}/test/_raise.html.erb"), e.file_name
  331. end
  332.  
  333. + def test_render_with_magic_and_errors
  334. + @view.render(:partial => "test/raise_at_3")
  335. + flunk "Render did not raise Template::Error"
  336. + rescue ActionView::Template::Error => e
  337. + assert_match %r!undefined local variable or method.*!, e.message
  338. + assert_equal "3", e.line_number
  339. + end
  340. +
  341. + def test_render_with_magic_and_errors_no_newline
  342. + @view.render(:partial => "test/raise_at_2_no_newline")
  343. + flunk "Render did not raise Template::Error"
  344. + rescue ActionView::Template::Error => e
  345. + assert_match %r!undefined local variable or method.*!, e.message
  346. + assert_equal "2", e.line_number
  347. + end
  348. +
  349. def test_render_sub_template_with_errors
  350. @view.render(:file => "test/sub_template_raise")
  351. flunk "Render did not raise Template::Error"
  352. @@ -281,9 +297,9 @@ class LazyViewRenderTest < ActiveSupport::TestCase
  353. if '1.9'.respond_to?(:force_encoding)
  354. def test_render_utf8_template_with_magic_comment
  355. with_external_encoding Encoding::ASCII_8BIT do
  356. - result = @view.render(:file => "test/utf8_magic.html.erb", :layouts => "layouts/yield")
  357. - assert_equal Encoding::UTF_8, result.encoding
  358. - assert_equal "Русский текст\n\nUTF-8\nUTF-8\nUTF-8\n", result
  359. + result = @view.render(:file => "test/_utf8_magic.html.erb", :layouts => "layouts/yield")
  360. + assert_equal Encoding::ASCII_8BIT, result.encoding
  361. + assert_equal "Русский текст\n\n", result.force_encoding('UTF-8')
  362. end
  363. end
  364.  
  365. @@ -291,7 +307,7 @@ class LazyViewRenderTest < ActiveSupport::TestCase
  366. with_external_encoding Encoding::UTF_8 do
  367. result = @view.render(:file => "test/utf8.html.erb", :layouts => "layouts/yield")
  368. assert_equal Encoding::UTF_8, result.encoding
  369. - assert_equal "Русский текст\n\nUTF-8\nUTF-8\nUTF-8\n", result
  370. + assert_equal "Русский текст\n\n", result
  371. end
  372. end
  373.  
  374. @@ -301,7 +317,7 @@ class LazyViewRenderTest < ActiveSupport::TestCase
  375. result = @view.render(:file => "test/utf8.html.erb", :layouts => "layouts/yield")
  376. flunk 'Should have raised incompatible encoding error'
  377. rescue ActionView::Template::Error => error
  378. - assert_match 'invalid byte sequence in Shift_JIS', error.original_exception.message
  379. + assert_match %r!.* on Shift_JIS!, error.original_exception.message
  380. end
  381. end
  382. end
  383. @@ -312,11 +328,34 @@ class LazyViewRenderTest < ActiveSupport::TestCase
  384. result = @view.render(:file => "test/utf8_magic_with_bare_partial.html.erb", :layouts => "layouts/yield")
  385. flunk 'Should have raised incompatible encoding error'
  386. rescue ActionView::Template::Error => error
  387. - assert_match 'invalid byte sequence in Shift_JIS', error.original_exception.message
  388. + assert_match %r!.* from Shift_JIS to UTF-8!, error.original_exception.message
  389. end
  390. end
  391. end
  392.  
  393. + unless Encoding.default_external == Encoding::US_ASCII
  394. + def test_render_with_multiple_encodings
  395. + result = @view.render(:file => "test/sjis_euc_partials.html.erb", :layouts => "layouts/yield")
  396. + assert_equal Encoding.default_external, result.encoding
  397. + with_external_encoding Encoding::UTF_8 do
  398. + assert_equal "日本語のテキスト\n日本語のテキスト\n\n".external_encode!, result.external_encode!
  399. + end
  400. + end
  401. +
  402. + def test_render_with_multiple_encodings_including_utf8
  403. + result = @view.render(:file => "test/sjis_euc_utf_partials.html.erb", :layouts => "layouts/yield")
  404. + assert_equal Encoding.default_external, result.encoding
  405. + assert_equal "日本語のテキスト\n日本語のテキスト\nРусский текст\n\n\n".external_encode!, result
  406. + end
  407. +
  408. + def test_render_euc_template_with_magic_comment
  409. + result = @view.render(:file => "test/_euc_jp_magic.html.erb", :layouts => "layouts/yield")
  410. + assert_equal Encoding.default_external, result.encoding
  411. + expected = "日本語のテキスト\n".encode!('euc-jp').external_encode!
  412. + assert_equal expected, result
  413. + end
  414. + end
  415. +
  416. def with_external_encoding(encoding)
  417. old, Encoding.default_external = Encoding.default_external, encoding
  418. yield
  419. --
  420. 1.6.3.3
Add Comment
Please, Sign In to add comment
Public Pastes
We use cookies for various purposes including analytics. By continuing to use Pastebin, you agree to our use of cookies as described in the Cookies Policy.  OK, I Understand
Not a member of Pastebin yet?
Sign Up, it unlocks many cool features!