Changeset 5224

Timestamp:

10/04/06 09:03:57 (2 years ago)

Author:

nzkoz

Message:

Pull in latest multibyte patch. Closes #6346 [Manfred Stienstra]

Files:

• trunk/activesupport/CHANGELOG (modified) (1 diff)
• trunk/activesupport/lib/active_support/multibyte/generators/generate_tables.rb (modified) (4 diffs)
• trunk/activesupport/lib/active_support/multibyte/handlers/utf8_handler.rb (modified) (4 diffs)
• trunk/activesupport/lib/active_support/values/unicode_tables-1.8.4.dat (deleted)
• trunk/activesupport/lib/active_support/values/unicode_tables-1.8.5.dat (deleted)
• trunk/activesupport/lib/active_support/values/unicode_tables.dat (added)
• trunk/activesupport/test/multibyte_chars_test.rb (modified) (1 diff)
• trunk/activesupport/test/multibyte_handler_test.rb (modified) (1 diff)

trunk/activesupport/CHANGELOG

@@ -1 +1 @@
-*SVN*
-
-
+
+
+
-
-* Fix issue with #class_inheritable_accessor saving updates to the parent class when initialized with an Array or Hash [mojombo]

trunk/activesupport/lib/active_support/multibyte/generators/generate_tables.rb

@@ -19 +19 @@
-      :codepoints => BASE_URI + 'UnicodeData.txt',
-      :composition_exclusion => BASE_URI + 'CompositionExclusions.txt',
-
+
+
-    }
-    
@@ -34 +35 @@
-      @ucd.composition_map = {}
-      @ucd.boundary = {}
+      @ucd.cp1252 = {}
-    end
-    
@@ -88 +90 @@
-    end
-    
+    def parse_cp1252(line)
+      if line =~ /^([0-9A-Fx]+)\s([0-9A-Fx]+)/i
+        @ucd.cp1252[$1.hex] = $2.hex
+      end
+    end
+    
-    def create_composition_map
-      @ucd.codepoints.each do |_, cp|
@@ -126 +134 @@
-    def dump_to(filename)
-      File.open(filename, 'wb') do |f|
-
+, @ucd.cp1252
-      end
-    end

trunk/activesupport/lib/active_support/multibyte/handlers/utf8_handler.rb

@@ -8 +8 @@
-  
-  class UnicodeDatabase #:nodoc:
-
+, :cp1252
-    
-    # Creates a new UnicodeDatabase instance and loads the database.
-    def initialize
-      begin
-
+, @cp1252
-      rescue Exception => e
-          raise IOError.new("Couldn't load the unicode tables for UTF8Handler (#{e.message}), handler is unusable")
@@ -21 +21 @@
-      @composition_map ||= {}
-      @boundary ||= {}
+      @cp1252 ||= {}
-      
-      # Redefine the === method so we can write shorter rules for grapheme cluster breaks
@@ -42 +43 @@
-    # Returns the filename for the data file for this version
-    def self.filename
--#{VERSION}
+
-    end
-    
-    # Loads the unicode database and returns all the internal objects of UnicodeDatabase
-    def self.load
-      begin
-        return load_file(filename)
-      rescue Exception
-        # If we can't load our own version, try the rest
-        Dir["#{dirname}/*.dat"].sort.each do |dat|
-          begin
-            return load_file(dat)
-          rescue Exception
-          end
-        end
-      end
-      raise IOError.new("Can't load a marshal file for your version of Ruby")
-    end
-    
-    def self.load_file(filename)
-      File.open(self.filename, 'rb') { |f| Marshal.load f.read }
-    end
@@ -276 +262 @@
-      # Strips all the non-utf-8 bytes from the string resulting in a valid utf-8 string
-      def tidy_bytes(str)
-
+
+
+
+
+
-      end
-      

trunk/activesupport/test/multibyte_chars_test.rb

@@ -140 +140 @@
-  def test_resilience
-    assert_nothing_raised do
-1, @s[:bytes].chars.size, "There's only one valid utf-8 byte in the string
+5, @s[:bytes].chars.size, "The sequence contains five interpretable bytes
-    end
+    reversed = [0xb8, 0x17e, 0x8, 0x2c6, 0xa5].reverse.pack('U*')
-    assert_nothing_raised do
-"\010", @s[:bytes].chars.reverse, "There's only one valid utf-8 byte in the string
+reversed, @s[:bytes].chars.reverse.to_s, "Reversing the string should only yield interpretable bytes
-    end
-    assert_nothing_raised do
-      @s[:bytes].chars.reverse!
-"\010", @s[:bytes], "There's only one valid utf-8 byte in the string
+reversed, @s[:bytes].to_s, "Reversing the string should only yield interpretable bytes
-    end
-  end

trunk/activesupport/test/multibyte_handler_test.rb

@@ -225 +225 @@
-  
-  def test_tidy_bytes
-
-
+
+
+
-    assert_nothing_raised { @handler.tidy_bytes(@bytestring).unpack('U*') }
+    
+    assert_equal "\xC3\xA7", @handler.tidy_bytes("\xE7") # iso_8859_1: small c cedilla
+    assert_equal "\xC2\xA9", @handler.tidy_bytes("\xA9") # iso_8859_1: copyright symbol
+    assert_equal "\xE2\x80\x9C", @handler.tidy_bytes("\x93") # win_1252: left smart quote
+    assert_equal "\xE2\x82\xAC", @handler.tidy_bytes("\x80") # win_1252: euro
+    assert_equal "\x00", @handler.tidy_bytes("\x00") # null char
+    assert_equal [0xef, 0xbf, 0xbd].pack('U*'), @handler.tidy_bytes("\xef\xbf\xbd") # invalid char
-  end
-