Tcl Source Code

Faster UTF encoding
Login

The apn-utf-fastpath branch introduces some changes to the UtfToUtfProc function for improved performance. This function is called to convert standard UTF-8 to and from Tcl's internal modified UTF-8. It is invoked through the encoding convert* commands or by channel I/O when the channel encoding is set to utf-8 (which is almost always the case on modern systems). Channels configured for other encodings are unaffected.

The benchmarks below are based on the following categories of languages with proportion of ASCII in the content based on the language (as per the Internet)

The benchmark table shows timings when run against content based on the statistics above as well as pathological worst case where ASCII characters alternate with non-ASCII characters. As shown in the table,

On the flip side, for pathological cases (where characters alternate between the ASCII fastpath and normal processing) there is no marked difference.

Benchmarks

The I/O benchmarks reflect only the time for encoding, not Tcl command, channel overhead etc.

(times in microseconds)

apn-utf-fastpath branch...

Linux (WSL gcc 11)

apnadkarni@IO:/mnt/c/src/tcltk/wip/tcl/unix/build/22$ ./tcltest /mnt/c/src/tcltk/wip/tcl/win/utfbench.tcl 10000
Calibrating...0.035229554990273984 µs/#-overhead 0.035313 µs/# 55277069 # 28318170 #/sec

                                      convertto  convertfrom         puts         read
 ascii (10000)                            7.543        7.396        6.791        7.531
 western (10000)                         13.424       13.878       12.740       14.240
 cyrillic (10000)                        87.151       91.546       86.290       95.333
 eastern (10000)                        122.641      115.172      120.344      115.892
 western - pathological (10000)          67.377       71.195       66.244       74.391
 cyrillic - pathological (10000)         67.678       71.685       66.209       74.355
 eastern - pathological (10000)          72.026       73.623       70.780       77.394

Windows VS2022

c:\src\tcltk\wip\tcl\win>Release_AMD64_VC1944\tcltest91.exe c:/src/tcltk/wip/tcl/win/utfbench.tcl 10000
Calibrating...0.035684211570548505 µs/#-overhead 0.037354 µs/# 52256094 # 26770539 #/sec

                                      convertto  convertfrom         puts         read
 ascii (10000)                            7.033        6.836        6.358        7.191
 western (10000)                         12.524       12.762       11.840       12.910
 cyrillic (10000)                        65.135       65.291       63.650       65.360
 eastern (10000)                         82.112       81.329       81.534       88.101
 western - pathological (10000)          56.760       56.959       55.262       55.981
 cyrillic - pathological (10000)         56.568       56.792       55.627       56.974
 eastern - pathological (10000)          62.154       62.200       62.244       63.942

Current trunk

Linux (WSL gcc 11)

apnadkarni@IO:/mnt/c/src/tcltk/main/tcl/unix/build/22$ ./tcltest /mnt/c/src/tcltk/wip/tcl/win/utfbench.tcl 10000
Calibrating...0.03797365585219985 µs/#-overhead 0.038160 µs/# 51154602 # 26205499 #/sec

                                      convertto  convertfrom         puts         read
 ascii (10000)                           23.296       23.203       22.695       23.248
 western (10000)                         27.787       27.604       26.921       28.032
 cyrillic (10000)                       114.211      114.608      111.627      115.822
 eastern (10000)                        135.507      134.657      134.713      139.039
 western - pathological (10000)          81.181       80.269       78.885       81.941
 cyrillic - pathological (10000)         80.334       80.369       80.079       82.513
 eastern - pathological (10000)          78.478       77.336       77.329       79.741

Windows

c:\src\tcltk\main\tcl\win>Release_AMD64_VC1944\tcltest91.exe c:/src/tcltk/wip/tcl/win/utfbench.tcl  10000
Calibrating...0.035101157673175384 µs/#-overhead 0.035233 µs/# 55402464 # 28382409 #/sec

                                      convertto  convertfrom         puts         read
 ascii (10000)                           24.412       24.183       23.481       24.450
 western (10000)                         27.331       27.320       26.369       27.893
 cyrillic (10000)                        74.042       77.117       71.627       77.692
 eastern (10000)                         93.492      100.079       92.841      101.378
 western - pathological (10000)          54.441       55.761       53.439       59.602
 cyrillic - pathological (10000)         54.644       55.910       53.274       57.113
 eastern - pathological (10000)          60.719       64.458       59.521       63.114

The benchmark itself is below:

puts -nonewline "Calibrating..."
flush stdout
puts [timerate -calibrate {}]
puts ""

proc format_timerate {timerate} {
    return [format %12.3f [lindex $timerate 0]]
}

proc measure {lang len alternate} {
    global texts

    if {$alternate} {
        # Alternate ascii with language-specific character
        set text [string cat X [string index $texts($lang) end]]
        set label "$lang - pathological"
    } else {
        set text $texts($lang)
        set label $lang
    }
    set repetitions [expr {$len/[string length $text]}]
    set text [string repeat $text $repetitions]
    set utf8 [encoding convertto utf-8 $text]
    puts -nonewline [format "%-35s" " $label ($len)"]
    puts -nonewline [format_timerate \
                         [timerate {encoding convertto utf-8 $text}]]
    puts -nonewline \
        [string cat \
            " " \
    	    [format_timerate [timerate {encoding convertfrom utf-8 $utf8}]]]

    if {[llength [info commands testchancreate]]} {
        set chan [testchancreate sink]
        fconfigure $chan -encoding utf-8 -buffering none
        puts -nonewline \
            [string cat \
                " " \
                [format_timerate [timerate {puts -nonewline $chan $text}]]]
        close $chan
        set chan [testchancreate source $utf8]
        fconfigure $chan -encoding utf-8 -buffering none
        puts -nonewline \
            [string cat \
                " " \
                [format_timerate [timerate {read $chan $len}]]]
        close $chan
    }

    puts ""
}

set lengths [expr {[llength $argv] ? $argv : [list 100 1000 10000]}]

# Basic text patterns. Last character must be non-ASCII
set texts(ascii) xy
# Western Europe >95% ASCII
set texts(western) 0123456789012345678\xe9
# Far Eastern, almost no ASCII
set texts(eastern) \u4e4e
# Cyrillic, 20% - mostly punctuation
set texts(cyrillic) x\u0411\u0411\u0411\u0411

puts [format "%35s%12s %12s %12s %12s" " " convertto convertfrom puts read]

# Measure "statistical sample"
foreach lang {ascii western cyrillic eastern} {
    foreach len $lengths {
        measure $lang $len 0
    }
}

# Measure "pathological" case where ascii,non-ascii alternate
foreach lang {ascii western cyrillic eastern} {
    if {$lang ne "ascii"} {
        foreach len $lengths {
            measure $lang $len 1
        }
    }
}