Tcl Source Code

Check-in [67baae2829]
Login

Many hyperlinks are disabled.
Use anonymous login to enable hyperlinks.

Overview
Comment:Fix for [0cd1ae596e709259], under strict encoding, [gets] returns an error even though a complete line is available.
Downloads: Tarball | ZIP archive | SQL archive
Timelines: family | ancestors | descendants | both | trunk | main
Files: files | file ages | folders
SHA3-256: 67baae2829603703e8fcd8c39aa2b32b1b0d2b547bedeeb75e0c428d70e3ec5c
User & Date: pooryorick 2023-04-14 07:21:48
References
2023-04-18
23:57
Merge trunk [67baae2829]. check-in: f63a6c9a50 user: pooryorick tags: trunk-encodingdefaultstrict
2023-04-14
07:23 Ticket [0cd1ae596e] Under strict encoding, gets returns an error even though a complete line is available status still Open with 5 other changes artifact: 1d05229fee user: pooryorick
Context
2023-04-18
23:57
Merge trunk [67baae2829]. check-in: f63a6c9a50 user: pooryorick tags: trunk-encodingdefaultstrict
2023-04-14
09:19
New test for [gets] with strict encoding. Also add "\" in io-76.9, missing in last commit. check-in: 567407c1cd user: pooryorick tags: trunk, main
07:21
Fix for [0cd1ae596e709259], under strict encoding, [gets] returns an error even though a complete li... check-in: 67baae2829 user: pooryorick tags: trunk, main
2023-04-13
16:58
Re-generate unix/configure check-in: 32d5cdd8d0 user: jan.nijtmans tags: trunk, main
2022-12-30
21:53
Add test for [gets] in non-strict mode after an encoding error. check-in: a4573db4f7 user: pooryorick tags: py-b8f575aa23
Changes
Hide Diffs Unified Diffs Ignore Whitespace Patch

Changes to generic/tclIO.c.

4993
4994
4995
4996
4997
4998
4999
5000
5001
5002
5003
5004
5005
5006
5007
5008
    if (chanPtr != statePtr->topChanPtr) {
	TclChannelRelease((Tcl_Channel)chanPtr);
	chanPtr = statePtr->topChanPtr;
	TclChannelPreserve((Tcl_Channel)chanPtr);
    }
    UpdateInterest(chanPtr);
    TclChannelRelease((Tcl_Channel)chanPtr);
    if (GotFlag(statePtr, CHANNEL_ENCODING_ERROR) &&
	    (copiedTotal == 0 || !GotFlag(statePtr, CHANNEL_NONBLOCKING))) {
	Tcl_SetErrno(EILSEQ);
	copiedTotal = -1;
    }
    return copiedTotal;
}

/*







|
<







4993
4994
4995
4996
4997
4998
4999
5000

5001
5002
5003
5004
5005
5006
5007
    if (chanPtr != statePtr->topChanPtr) {
	TclChannelRelease((Tcl_Channel)chanPtr);
	chanPtr = statePtr->topChanPtr;
	TclChannelPreserve((Tcl_Channel)chanPtr);
    }
    UpdateInterest(chanPtr);
    TclChannelRelease((Tcl_Channel)chanPtr);
    if (GotFlag(statePtr, CHANNEL_ENCODING_ERROR) && gs.bytesWrote == 0) {

	Tcl_SetErrno(EILSEQ);
	copiedTotal = -1;
    }
    return copiedTotal;
}

/*

Changes to tests/io.test.

9266
9267
9268
9269
9270
9271
9272
9273

9274
9275
9276
9277
9278
9279
9280
9281

9282

9283
9284
9285
9286
9287
9288
9289
9290

9291
9292
9293
9294
9295
9296
9297
9298
9299
9300
9301
9302
9303

9304
9305
9306
9307

9308
9309
9310
9311
9312
9313
9314
    set fn [makeFile {} io-75.6]
    set f [open $fn w+]
    fconfigure $f -encoding binary
    # \x81 is invalid in utf-8
    puts -nonewline $f A\x81
    flush $f
    seek $f 0
    fconfigure $f -encoding utf-8 -buffering none -eofchar "" -translation lf -profile strict

} -body {
    gets $f
} -cleanup {
    close $f
    removeFile io-75.6
} -match glob -returnCodes 1 -result {error reading "file*":\
	invalid or incomplete multibyte or wide character}


test io-75.7 {invalid utf-8 encoding gets is not ignored (-profile strict)} -setup {

    set fn [makeFile {} io-75.7]
    set f [open $fn w+]
    fconfigure $f -encoding binary
    # \x81 is invalid in utf-8
    puts -nonewline $f A\x81
    flush $f
    seek $f 0
    fconfigure $f -encoding utf-8 -buffering none -eofchar "" -translation lf -profile strict

} -body {
    read $f
} -cleanup {
    close $f
    removeFile io-75.7
} -match glob -returnCodes 1 -result {error reading "file*":\
    invalid or incomplete multibyte or wide character}

test io-75.8 {invalid utf-8 encoding eof handling (-profile strict)} -setup {
    set fn [makeFile {} io-75.8]
    set f [open $fn w+]
    fconfigure $f -encoding binary
    # \x81 is invalid in utf-8, but since \x1A comes first, -eofchar takes precedence.

    puts -nonewline $f A\x1A\x81
    flush $f
    seek $f 0
    fconfigure $f -encoding utf-8 -buffering none -eofchar \x1A -translation lf -profile strict

} -body {
    set d [read $f]
    binary scan $d H* hd
    lappend hd [eof $f]
    lappend hd [read $f]
    close $f
    set hd







|
>








>
|
>







|
>












|
>



|
>







9266
9267
9268
9269
9270
9271
9272
9273
9274
9275
9276
9277
9278
9279
9280
9281
9282
9283
9284
9285
9286
9287
9288
9289
9290
9291
9292
9293
9294
9295
9296
9297
9298
9299
9300
9301
9302
9303
9304
9305
9306
9307
9308
9309
9310
9311
9312
9313
9314
9315
9316
9317
9318
9319
9320
    set fn [makeFile {} io-75.6]
    set f [open $fn w+]
    fconfigure $f -encoding binary
    # \x81 is invalid in utf-8
    puts -nonewline $f A\x81
    flush $f
    seek $f 0
    fconfigure $f -encoding utf-8 -buffering none -eofchar {} \
	-translation lf -profile strict
} -body {
    gets $f
} -cleanup {
    close $f
    removeFile io-75.6
} -match glob -returnCodes 1 -result {error reading "file*":\
	invalid or incomplete multibyte or wide character}

test io-75.7 {
    invalid utf-8 encoding gets is not ignored (-profile strict)
} -setup {
    set fn [makeFile {} io-75.7]
    set f [open $fn w+]
    fconfigure $f -encoding binary
    # \x81 is invalid in utf-8
    puts -nonewline $f A\x81
    flush $f
    seek $f 0
    fconfigure $f -encoding utf-8 -buffering none -eofchar {} -translation lf \
	-profile strict
} -body {
    read $f
} -cleanup {
    close $f
    removeFile io-75.7
} -match glob -returnCodes 1 -result {error reading "file*":\
    invalid or incomplete multibyte or wide character}

test io-75.8 {invalid utf-8 encoding eof handling (-profile strict)} -setup {
    set fn [makeFile {} io-75.8]
    set f [open $fn w+]
    fconfigure $f -encoding binary
    # \x81 is invalid in utf-8, but since \x1A comes first, -eofchar takes
    # precedence.
    puts -nonewline $f A\x1A\x81
    flush $f
    seek $f 0
    fconfigure $f -encoding utf-8 -buffering none -eofchar \x1A \
	-translation lf -profile strict
} -body {
    set d [read $f]
    binary scan $d H* hd
    lappend hd [eof $f]
    lappend hd [read $f]
    close $f
    set hd
9324
9325
9326
9327
9328
9329
9330
9331

9332
9333
9334
9335
9336
9337
9338
    catch {puts -nonewline $f "A\u2022"} msg
    flush $f
    seek $f 0
    list [read $f] $msg
} -cleanup {
    close $f
    removeFile io-75.9
} -match glob -result [list {A} {error writing "*": invalid or incomplete multibyte or wide character}]


test io-75.10 {
    incomplete multibyte encoding read is not ignored because "binary" sets
    profile to strict
} -setup {
    set res {}
    set fn [makeFile {} io-75.10]







|
>







9330
9331
9332
9333
9334
9335
9336
9337
9338
9339
9340
9341
9342
9343
9344
9345
    catch {puts -nonewline $f "A\u2022"} msg
    flush $f
    seek $f 0
    list [read $f] $msg
} -cleanup {
    close $f
    removeFile io-75.9
} -match glob -result [list {A} {error writing "*":\
    invalid or incomplete multibyte or wide character}]

test io-75.10 {
    incomplete multibyte encoding read is not ignored because "binary" sets
    profile to strict
} -setup {
    set res {}
    set fn [makeFile {} io-75.10]
9368
9369
9370
9371
9372
9373
9374
9375

9376
9377
9378
9379
9380
9381
9382
    set f [open $fn w+]
    fconfigure $f -encoding binary
    # In shiftjis, \x81 starts a two-byte sequence.
    # But 2nd byte \xFF is not allowed
    puts -nonewline $f A\x81\xFFA
    flush $f
    seek $f 0
    fconfigure $f -encoding shiftjis -blocking 0 -eofchar "" -translation lf -profile strict

} -body {
    set d [read $f]
    binary scan $d H* hd
    lappend hd [catch {set d [read $f]} msg]
    lappend hd $msg
} -cleanup {
    close $f







|
>







9375
9376
9377
9378
9379
9380
9381
9382
9383
9384
9385
9386
9387
9388
9389
9390
    set f [open $fn w+]
    fconfigure $f -encoding binary
    # In shiftjis, \x81 starts a two-byte sequence.
    # But 2nd byte \xFF is not allowed
    puts -nonewline $f A\x81\xFFA
    flush $f
    seek $f 0
    fconfigure $f -encoding shiftjis -blocking 0 -eofchar {} -translation lf \
	-profile strict
} -body {
    set d [read $f]
    binary scan $d H* hd
    lappend hd [catch {set d [read $f]} msg]
    lappend hd $msg
} -cleanup {
    close $f
9391
9392
9393
9394
9395
9396
9397
9398
9399
9400
9401
9402
9403
9404
9405
    set res {}
    set fn [makeFile {} io-75.12]
    set f [open $fn w+]
    fconfigure $f -encoding binary
    puts -nonewline $f A\x81
    flush $f
    seek $f 0
    fconfigure $f -encoding utf-8 -buffering none -eofchar "" -translation lf
} -body {
    catch {read $f} errmsg
    lappend res $errmsg
    chan configure $f -profile tcl8
    seek $f 0
    set d [read $f]
    binary scan $d H* hd







|







9399
9400
9401
9402
9403
9404
9405
9406
9407
9408
9409
9410
9411
9412
9413
    set res {}
    set fn [makeFile {} io-75.12]
    set f [open $fn w+]
    fconfigure $f -encoding binary
    puts -nonewline $f A\x81
    flush $f
    seek $f 0
    fconfigure $f -encoding utf-8 -buffering none -eofchar {} -translation lf
} -body {
    catch {read $f} errmsg
    lappend res $errmsg
    chan configure $f -profile tcl8
    seek $f 0
    set d [read $f]
    binary scan $d H* hd
9419
9420
9421
9422
9423
9424
9425
9426

9427
9428
9429
9430
9431
9432
9433
9434
9435





























9436
9437
9438
9439
9440
9441
9442
    set fn [makeFile {} io-75.13]
    set f [open $fn w+]
    fconfigure $f -encoding binary
    # \x81 is invalid in utf-8
    puts -nonewline $f A\x81
    flush $f
    seek $f 0
    fconfigure $f -encoding utf-8 -blocking 0 -eofchar "" -translation lf -profile strict

} -body {
    set d [read $f]
    binary scan $d H* hd
    lappend hd [catch {read $f} msg]
    lappend hd $msg
} -cleanup {
    close $f
    removeFile io-75.13
} -match glob -result {41 1 {error reading "file*": invalid or incomplete multibyte or wide character}}






























# ### ### ### ######### ######### #########



test io-76.0 {channel modes} -setup {
    set datafile [makeFile {some characters} dummy]







|
>








|
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>







9427
9428
9429
9430
9431
9432
9433
9434
9435
9436
9437
9438
9439
9440
9441
9442
9443
9444
9445
9446
9447
9448
9449
9450
9451
9452
9453
9454
9455
9456
9457
9458
9459
9460
9461
9462
9463
9464
9465
9466
9467
9468
9469
9470
9471
9472
9473
9474
9475
9476
9477
9478
9479
9480
    set fn [makeFile {} io-75.13]
    set f [open $fn w+]
    fconfigure $f -encoding binary
    # \x81 is invalid in utf-8
    puts -nonewline $f A\x81
    flush $f
    seek $f 0
    fconfigure $f -encoding utf-8 -blocking 0 -eofchar {} -translation lf \
	-profile strict
} -body {
    set d [read $f]
    binary scan $d H* hd
    lappend hd [catch {read $f} msg]
    lappend hd $msg
} -cleanup {
    close $f
    removeFile io-75.13
} -match glob -result {41 1 {error reading "file*":\
    invalid or incomplete multibyte or wide character}}

test io-75.14 {
	[gets] succesfully returns lines prior to error

	invalid utf-8 encoding [gets] continues in non-strict mode after error
} -setup {
    set fn [makeFile {} io-75.14]
    set f [open $fn w+]
    fconfigure $f -encoding binary
    # \xc0 is invalid in utf-8
    puts -nonewline $f a\nb\xc0\nc\n
    flush $f
    seek $f 0
    fconfigure $f -encoding utf-8 -buffering none -eofchar {} \
	-translation lf -profile strict
} -body {
    lappend res [gets $f]
    set status [catch {gets $f} cres copts]
    lappend res $status $cres
    chan configure $f -profile tcl8 
    lappend res [gets $f]
    lappend res [gets $f]
    close $f
	return $res
} -cleanup {
    removeFile io-75.14
} -match glob -result {a 1 {error reading "*":\
    invalid or incomplete multibyte or wide character} bÀ c}

# ### ### ### ######### ######### #########



test io-76.0 {channel modes} -setup {
    set datafile [makeFile {some characters} dummy]
9483
9484
9485
9486
9487
9488
9489
9490

9491
9492
9493
9494
9495
9496
9497
9498
9499
9500
9501
9502
9503
9504
9505
9506
9507
9508
9509
9510
9511

9512
9513
9514
9515
9516
9517
9518
    set datafile [makeFile {some characters} dummy]
    set f [open $datafile r]
} -constraints testchannel -body {
    testchannel mremove-rd $f
} -returnCodes error -cleanup {
    close $f
    removeFile dummy
} -match glob -result {Tcl_RemoveChannelMode error: Bad mode, would make channel inacessible. Channel: "*"}


test io-76.5 {channel mode dropping} -setup {
    set datafile [makeFile {some characters} dummy]
    set f [open $datafile w]
} -constraints testchannel -body {
    testchannel mremove-rd $f
    list [testchannel mode $f] [testchannel maxmode $f]
} -cleanup {
    close $f
    removeFile dummy
} -result {{{} write} {{} write}}

test io-76.6 {channel mode dropping} -setup {
    set datafile [makeFile {some characters} dummy]
    set f [open $datafile w]
} -constraints testchannel -body {
    testchannel mremove-wr $f
} -returnCodes error -cleanup {
    close $f
    removeFile dummy
} -match glob -result {Tcl_RemoveChannelMode error: Bad mode, would make channel inacessible. Channel: "*"}


test io-76.7 {channel mode dropping} -setup {
    set datafile [makeFile {some characters} dummy]
    set f [open $datafile r+]
} -constraints testchannel -body {
    testchannel mremove-rd $f
    list [testchannel mode $f] [testchannel maxmode $f]







|
>




















|
>







9521
9522
9523
9524
9525
9526
9527
9528
9529
9530
9531
9532
9533
9534
9535
9536
9537
9538
9539
9540
9541
9542
9543
9544
9545
9546
9547
9548
9549
9550
9551
9552
9553
9554
9555
9556
9557
9558
    set datafile [makeFile {some characters} dummy]
    set f [open $datafile r]
} -constraints testchannel -body {
    testchannel mremove-rd $f
} -returnCodes error -cleanup {
    close $f
    removeFile dummy
} -match glob -result {Tcl_RemoveChannelMode error:\
    Bad mode, would make channel inacessible. Channel: "*"}

test io-76.5 {channel mode dropping} -setup {
    set datafile [makeFile {some characters} dummy]
    set f [open $datafile w]
} -constraints testchannel -body {
    testchannel mremove-rd $f
    list [testchannel mode $f] [testchannel maxmode $f]
} -cleanup {
    close $f
    removeFile dummy
} -result {{{} write} {{} write}}

test io-76.6 {channel mode dropping} -setup {
    set datafile [makeFile {some characters} dummy]
    set f [open $datafile w]
} -constraints testchannel -body {
    testchannel mremove-wr $f
} -returnCodes error -cleanup {
    close $f
    removeFile dummy
} -match glob -result {Tcl_RemoveChannelMode error:\
    Bad mode, would make channel inacessible. Channel: "*"}

test io-76.7 {channel mode dropping} -setup {
    set datafile [makeFile {some characters} dummy]
    set f [open $datafile r+]
} -constraints testchannel -body {
    testchannel mremove-rd $f
    list [testchannel mode $f] [testchannel maxmode $f]
9537
9538
9539
9540
9541
9542
9543
9544

9545
9546
9547
9548
9549
9550
9551
9552
9553
9554
9555

9556
9557
9558
9559
9560
9561
9562
9563
9564
9565
    set f [open $datafile r+]
} -constraints testchannel -body {
    testchannel mremove-wr $f
    testchannel mremove-rd $f
} -returnCodes error -cleanup {
    close $f
    removeFile dummy
} -match glob -result {Tcl_RemoveChannelMode error: Bad mode, would make channel inacessible. Channel: "*"}


test io-76.10 {channel mode dropping} -setup {
    set datafile [makeFile {some characters} dummy]
    set f [open $datafile r+]
} -constraints testchannel -body {
    testchannel mremove-rd $f
    testchannel mremove-wr $f
} -returnCodes error -cleanup {
    close $f
    removeFile dummy
} -match glob -result {Tcl_RemoveChannelMode error: Bad mode, would make channel inacessible. Channel: "*"}


# cleanup
foreach file [list fooBar longfile script script2 output test1 pipe my_script \
	test2 test3 cat stdout kyrillic.txt utf8-fcopy.txt utf8-rp.txt] {
    removeFile $file
}
cleanupTests
}
namespace delete ::tcl::test::io
return







|
>










|
>










9577
9578
9579
9580
9581
9582
9583
9584
9585
9586
9587
9588
9589
9590
9591
9592
9593
9594
9595
9596
9597
9598
9599
9600
9601
9602
9603
9604
9605
9606
9607
    set f [open $datafile r+]
} -constraints testchannel -body {
    testchannel mremove-wr $f
    testchannel mremove-rd $f
} -returnCodes error -cleanup {
    close $f
    removeFile dummy
} -match glob -result {Tcl_RemoveChannelMode error:
    Bad mode, would make channel inacessible. Channel: "*"}

test io-76.10 {channel mode dropping} -setup {
    set datafile [makeFile {some characters} dummy]
    set f [open $datafile r+]
} -constraints testchannel -body {
    testchannel mremove-rd $f
    testchannel mremove-wr $f
} -returnCodes error -cleanup {
    close $f
    removeFile dummy
} -match glob -result {Tcl_RemoveChannelMode error:\
    Bad mode, would make channel inacessible. Channel: "*"}

# cleanup
foreach file [list fooBar longfile script script2 output test1 pipe my_script \
	test2 test3 cat stdout kyrillic.txt utf8-fcopy.txt utf8-rp.txt] {
    removeFile $file
}
cleanupTests
}
namespace delete ::tcl::test::io
return