+++ /dev/null
--- Provides UTF-8 aware string functions implemented in pure lua:
--- * string.utf8len(s)
--- * string.utf8sub(s, i, j)
---
--- All functions behave as their non UTF-8 aware counterparts with the exception
--- that UTF-8 characters are used instead of bytes for all units.
---
--- Note: all validations had been removed due to awesome usage specifics.
---[[
-Copyright (c) 2006-2007, Kyle Smith
-Modified by Alexander Yakushev, 2010-2013.
-All rights reserved.
-
-Redistribution and use in source and binary forms, with or without
-modification, are permitted provided that the following conditions are met:
-
-* Redistributions of source code must retain the above copyright notice,
-this list of conditions and the following disclaimer.
-* Redistributions in binary form must reproduce the above copyright
-notice, this list of conditions and the following disclaimer in the
-documentation and/or other materials provided with the distribution.
-* Neither the name of the author nor the names of its contributors may be
-used to endorse or promote products derived from this software without
-specific prior written permission.
-
-THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
-AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
-IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
-DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE
-FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
-DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
-SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
-CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
-OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
-OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
---]]
-
--- ABNF from RFC 3629
---
--- UTF8-octets = *( UTF8-char )
--- UTF8-char = UTF8-1 / UTF8-2 / UTF8-3 / UTF8-4
--- UTF8-1 = %x00-7F
--- UTF8-2 = %xC2-DF UTF8-tail
--- UTF8-3 = %xE0 %xA0-BF UTF8-tail / %xE1-EC 2( UTF8-tail ) /
--- %xED %x80-9F UTF8-tail / %xEE-EF 2( UTF8-tail )
--- UTF8-4 = %xF0 %x90-BF 2( UTF8-tail ) / %xF1-F3 3( UTF8-tail ) /
--- %xF4 %x80-8F 2( UTF8-tail )
--- UTF8-tail = %x80-BF
---
-
--- returns the number of bytes used by the UTF-8 character at byte i in s
--- also doubles as a UTF-8 character validator
-
-local utf8 = {}
-
-function utf8.charbytes (s, i)
- -- argument defaults
- i = i or 1
- local c = string.byte(s, i)
-
- -- determine bytes needed for character, based on RFC 3629
- if c > 0 and c <= 127 then
- -- UTF8-1
- return 1
- elseif c >= 194 and c <= 223 then
- -- UTF8-2
- local c2 = string.byte(s, i + 1)
- return 2
- elseif c >= 224 and c <= 239 then
- -- UTF8-3
- local c2 = s:byte(i + 1)
- local c3 = s:byte(i + 2)
- return 3
- elseif c >= 240 and c <= 244 then
- -- UTF8-4
- local c2 = s:byte(i + 1)
- local c3 = s:byte(i + 2)
- local c4 = s:byte(i + 3)
- return 4
- end
-end
-
--- returns the number of characters in a UTF-8 string
-function utf8.len (s)
- local pos = 1
- local bytes = string.len(s)
- local len = 0
-
- while pos <= bytes and len ~= chars do
- local c = string.byte(s,pos)
- len = len + 1
-
- pos = pos + utf8.charbytes(s, pos)
- end
-
- if chars ~= nil then
- return pos - 1
- end
-
- return len
-end
-
--- functions identically to string.sub except that i and j are UTF-8 characters
--- instead of bytes
-function utf8.sub (s, i, j)
- j = j or -1
-
- if i == nil then
- return ""
- end
-
- local pos = 1
- local bytes = string.len(s)
- local len = 0
-
- -- only set l if i or j is negative
- local l = (i >= 0 and j >= 0) or utf8.len(s)
- local startChar = (i >= 0) and i or l + i + 1
- local endChar = (j >= 0) and j or l + j + 1
-
- -- can't have start before end!
- if startChar > endChar then
- return ""
- end
-
- -- byte offsets to pass to string.sub
- local startByte, endByte = 1, bytes
-
- while pos <= bytes do
- len = len + 1
-
- if len == startChar then
- startByte = pos
- end
-
- pos = pos + utf8.charbytes(s, pos)
-
- if len == endChar then
- endByte = pos - 1
- break
- end
- end
-
- return string.sub(s, startByte, endByte)
-end
-
--- replace UTF-8 characters based on a mapping table
-function utf8.replace (s, mapping)
- local pos = 1
- local bytes = string.len(s)
- local charbytes
- local newstr = ""
-
- while pos <= bytes do
- charbytes = utf8.charbytes(s, pos)
- local c = string.sub(s, pos, pos + charbytes - 1)
- newstr = newstr .. (mapping[c] or c)
- pos = pos + charbytes
- end
-
- return newstr
-end
-
-return utf8
repos / dotfiles.git / blobdiff