hexo/node_modules/js-tiktoken/dist/index.js

import { never, Tiktoken, getEncodingNameForModel } from './chunk-P6KNE6HN.js';
export { Tiktoken, getEncodingNameForModel } from './chunk-P6KNE6HN.js';

// src/ranks/gpt2.js
var gpt2_default = { "explicit_n_vocab": 50257, "pat_str": "'s|'t|'re|'ve|'m|'ll|'d| ?\\p{L}+| ?\\p{N}+| ?[^\\s\\p{L}\\p{N}]+|\\s+(?!\\S)|\\s+", "special_tokens": { "<|endoftext|>": 50256 }, "bpe_ranks": "! 0 IQ== Ig== Iw== JA== JQ== Jg== Jw== KA== KQ== Kg== Kw== LA== LQ== Lg== Lw== MA== MQ== Mg== Mw== NA== NQ== Ng== Nw== OA== OQ== Og== Ow== PA== PQ== Pg== Pw== QA== QQ== Qg== Qw== RA== RQ== Rg== Rw== SA== SQ== Sg== Sw== TA== TQ== Tg== Tw== UA== UQ== Ug== Uw== VA== VQ== Vg== Vw== WA== WQ== Wg== Ww== XA== XQ== Xg== Xw== YA== YQ== Yg== Yw== ZA== ZQ== Zg== Zw== aA== aQ== ag== aw== bA== bQ== bg== bw== cA== cQ== cg== cw== dA== dQ== dg== dw== eA== eQ== eg== ew== fA== fQ== fg== oQ== og== ow== pA== pQ== pg== pw== qA== qQ== qg== qw== rA== rg== rw== sA== sQ== sg== sw== tA== tQ== tg== tw== uA== uQ== ug== uw== vA== vQ== vg== vw== wA== wQ== wg== ww== xA== xQ== xg== xw== yA== yQ== yg== yw== zA== zQ== zg== zw== 0A== 0Q== 0g== 0w== 1A== 1Q== 1g== 1w== 2A== 2Q== 2g== 2w== 3A== 3Q== 3g== 3w== 4A== 4Q== 4g== 4w== 5A== 5Q== 5g== 5w== 6A== 6Q== 6g== 6w== 7A== 7Q== 7g== 7w== 8A== 8Q== 8g== 8w== 9A== 9Q== 9g== 9w== +A== +Q== +g== +w== /A== /Q== /g== /w== AA== AQ== Ag== Aw== BA== BQ== Bg== Bw== CA== CQ== Cg== Cw== DA== DQ== Dg== Dw== EA== EQ== Eg== Ew== FA== FQ== Fg== Fw== GA== GQ== Gg== Gw== HA== HQ== Hg== Hw== IA== fw== gA== gQ== gg== gw== hA== hQ== hg== hw== iA== iQ== ig== iw== jA== jQ== jg== jw== kA== kQ== kg== kw== lA== lQ== lg== lw== mA== mQ== mg== mw== nA== nQ== ng== nw== oA== rQ== IHQ= IGE= aGU= aW4= cmU= b24= IHRoZQ== ZXI= IHM= YXQ= IHc= IG8= ZW4= IGM= aXQ= aXM= YW4= b3I= ZXM= IGI= ZWQ= IGY= aW5n IHA= b3U= IGFu YWw= YXI= IHRv IG0= IG9m IGlu IGQ= IGg= IGFuZA== aWM= YXM= bGU= IHRo aW9u b20= bGw= ZW50 IG4= IGw= c3Q= IHJl dmU= IGU= cm8= bHk= IGJl IGc= IFQ= Y3Q= IFM= aWQ= b3Q= IEk= dXQ= ZXQ= IEE= IGlz IG9u aW0= YW0= b3c= YXk= YWQ= c2U= IHRoYXQ= IEM= aWc= IGZvcg== YWM= IHk= dmVy dXI= IHU= bGQ= IHN0 IE0= J3M= IGhl IGl0 YXRpb24= aXRo aXI= Y2U= IHlvdQ== aWw= IEI= IHdo b2w= IFA= IHdpdGg= IDE= dGVy Y2g= IGFz IHdl ICg= bmQ= aWxs IEQ= aWY= IDI= YWc= ZXJz a2U= ICI= IEg= ZW0= IGNvbg== IFc= IFI= aGVy IHdhcw== IHI= b2Q= IEY= dWw= YXRl IGF0 cmk= cHA= b3Jl IFRoZQ== IHNl dXM= IHBybw== IGhh dW0= IGFyZQ== IGRl YWlu YW5k IG9y aWdo ZXN0 aXN0 YWI= cm9t IE4= dGg= IGNvbQ== IEc= dW4= b3A= MDA= IEw= IG5vdA== ZXNz IGV4 IHY= cmVz IEU= ZXc= aXR5 YW50 IGJ5 ZWw= b3M= b3J0 b2M= cXU= IGZyb20= IGhhdmU= IHN1 aXZl b3VsZA== IHNo IHRoaXM= bnQ= cmE= cGU= aWdodA== YXJ0 bWVudA== IGFs dXN0 ZW5k LS0= YWxs IE8= YWNr IGNo IGxl aWVz cmVk YXJk 4oA= b3V0 IEo= IGFi ZWFy aXY= YWxseQ== b3Vy b3N0 Z2g= cHQ= IHBs YXN0 IGNhbg== YWs= b21l dWQ= VGhl IGhpcw== IGRv IGdv IGhhcw== Z2U= J3Q= IFU= cm91 IHNh IGo= IGJ1dA== IHdvcg== IGFsbA== ZWN0 IGs= YW1l IHdpbGw= b2s= IHdoZQ== IHRoZXk= aWRl MDE= ZmY= aWNo cGw= dGhlcg== IHRy Li4= IGludA== aWU= dXJl YWdl IG5l aWFs YXA= aW5l aWNl IG1l IG91dA== YW5z b25l b25n aW9ucw== IHdobw== IEs= IHVw IHRoZWly IGFk IDM= IHVz YXRlZA== b3Vz IG1vcmU= dWU= b2c= IFN0 aW5k aWtl IHNv aW1l cGVy LiI= YmVy aXo= YWN0 IG9uZQ== IHNhaWQ= IC0= YXJl IHlvdXI= Y2M= IFRo IGNs ZXA= YWtl YWJsZQ== aXA= IGNvbnQ= IHdoaWNo aWE= IGlt IGFib3V0 IHdlcmU= dmVyeQ== dWI= IGhhZA== IGVu IGNvbXA= LCI= IElu IHVu IGFn aXJl YWNl YXU= YXJ5 IHdvdWxk YXNz cnk= IOKA Y2w= b29r ZXJl c28= IFY= aWdu aWI= IG9mZg== IHRl dmVu IFk= aWxl b3Nl aXRl b3Jt IDIwMQ== IHJlcw== IG1hbg== IHBlcg== IG90aGVy b3Jk dWx0 IGJlZW4= IGxpa2U= YXNl YW5jZQ== a3M= YXlz b3du ZW5jZQ== IGRpcw== Y3Rpb24= IGFueQ== IGFwcA== IHNw aW50 cmVzcw== YXRpb25z YWls IDQ= aWNhbA== IHRoZW0= IGhlcg== b3VudA== IENo IGFy IGlm IHRoZXJl IHBl IHllYXI= YXY= IG15 IHNvbWU= IHdoZW4= b3VnaA== YWNo IHRoYW4= cnU= b25k aWNr IG92ZXI= dmVs IHF1 Cgo= IHNj cmVhdA== cmVl IEl0 b3VuZA== cG9ydA== IGFsc28= IHBhcnQ= ZnRlcg== IGtu IGJlYw== IHRpbWU= ZW5z IDU= b3BsZQ== IHdoYXQ= IG5v ZHU= bWVy YW5n IG5ldw== LS0tLQ== IGdldA== b3J5 aXRpb24= aW5ncw== IGp1c3Q= IGludG8= IDA= ZW50cw== b3Zl dGU= IHBlb3BsZQ== IHByZQ== IGl0cw== IHJlYw== IHR3 aWFu aXJzdA== YXJr b3Jz IHdvcms= YWRl b2I= IHNoZQ== IG91cg== d24= aW5r bGlj IDE5 IEhl aXNo bmRlcg== YXVzZQ== IGhpbQ== b25z IFs= IHJv Zm9ybQ== aWxk YXRlcw== 

// src/ranks/p50k_base.js
var p50k_base_default = { "explicit_n_vocab": 50281, "pat_str": "'s|'t|'re|'ve|'m|'ll|'d| ?\\p{L}+| ?\\p{N}+| ?[^\\s\\p{L}\\p{N}]+|\\s+(?!\\S)|\\s+", "special_tokens": { "<|endoftext|>": 50256 }, "bpe_ranks": "! 0 IQ== Ig== Iw== JA== JQ== Jg== Jw== KA== KQ== Kg== Kw== LA== LQ== Lg== Lw== MA== MQ== Mg== Mw== NA== NQ== Ng== Nw== OA== OQ== Og== Ow== PA== PQ== Pg== Pw== QA== QQ== Qg== Qw== RA== RQ== Rg== Rw== SA== SQ== Sg== Sw== TA== TQ== Tg== Tw== UA== UQ== Ug== Uw== VA== VQ== Vg== Vw== WA== WQ== Wg== Ww== XA== XQ== Xg== Xw== YA== YQ== Yg== Yw== ZA== ZQ== Zg== Zw== aA== aQ== ag== aw== bA== bQ== bg== bw== cA== cQ== cg== cw== dA== dQ== dg== dw== eA== eQ== eg== ew== fA== fQ== fg== oQ== og== ow== pA== pQ== pg== pw== qA== qQ== qg== qw== rA== rg== rw== sA== sQ== sg== sw== tA== tQ== tg== tw== uA== uQ== ug== uw== vA== vQ== vg== vw== wA== wQ== wg== ww== xA== xQ== xg== xw== yA== yQ== yg== yw== zA== zQ== zg== zw== 0A== 0Q== 0g== 0w== 1A== 1Q== 1g== 1w== 2A== 2Q== 2g== 2w== 3A== 3Q== 3g== 3w== 4A== 4Q== 4g== 4w== 5A== 5Q== 5g== 5w== 6A== 6Q== 6g== 6w== 7A== 7Q== 7g== 7w== 8A== 8Q== 8g== 8w== 9A== 9Q== 9g== 9w== +A== +Q== +g== +w== /A== /Q== /g== /w== AA== AQ== Ag== Aw== BA== BQ== Bg== Bw== CA== CQ== Cg== Cw== DA== DQ== Dg== Dw== EA== EQ== Eg== Ew== FA== FQ== Fg== Fw== GA== GQ== Gg== Gw== HA== HQ== Hg== Hw== IA== fw== gA== gQ== gg== gw== hA== hQ== hg== hw== iA== iQ== ig== iw== jA== jQ== jg== jw== kA== kQ== kg== kw== lA== lQ== lg== lw== mA== mQ== mg== mw== nA== nQ== ng== nw== oA== rQ== IHQ= IGE= aGU= aW4= cmU= b24= IHRoZQ== ZXI= IHM= YXQ= IHc= IG8= ZW4= IGM= aXQ= aXM= YW4= b3I= ZXM= IGI= ZWQ= IGY= aW5n IHA= b3U= IGFu YWw= YXI= IHRv IG0= IG9m IGlu IGQ= IGg= IGFuZA== aWM= YXM= bGU= IHRo aW9u b20= bGw= ZW50 IG4= IGw= c3Q= IHJl dmU= IGU= cm8= bHk= IGJl IGc= IFQ= Y3Q= IFM= aWQ= b3Q= IEk= dXQ= ZXQ= IEE= IGlz IG9u aW0= YW0= b3c= YXk= YWQ= c2U= IHRoYXQ= IEM= aWc= IGZvcg== YWM= IHk= dmVy dXI= IHU= bGQ= IHN0 IE0= J3M= IGhl IGl0 YXRpb24= aXRo aXI= Y2U= IHlvdQ== aWw= IEI= IHdo b2w= IFA= IHdpdGg= IDE= dGVy Y2g= IGFz IHdl ICg= bmQ= aWxs IEQ= aWY= IDI= YWc= ZXJz a2U= ICI= IEg= ZW0= IGNvbg== IFc= IFI= aGVy IHdhcw== IHI= b2Q= IEY= dWw= YXRl IGF0 cmk= cHA= b3Jl IFRoZQ== IHNl dXM= IHBybw== IGhh dW0= IGFyZQ== IGRl YWlu YW5k IG9y aWdo ZXN0 aXN0 YWI= cm9t IE4= dGg= IGNvbQ== IEc= dW4= b3A= MDA= IEw= IG5vdA== ZXNz IGV4 IHY= cmVz IEU= ZXc= aXR5 YW50 IGJ5 ZWw= b3M= b3J0 b2M= cXU= IGZyb20= IGhhdmU= IHN1 aXZl b3VsZA== IHNo IHRoaXM= bnQ= cmE= cGU= aWdodA== YXJ0 bWVudA== IGFs dXN0 ZW5k LS0= YWxs IE8= YWNr IGNo IGxl aWVz cmVk YXJk 4oA= b3V0 IEo= IGFi ZWFy aXY= YWxseQ== b3Vy b3N0 Z2g= cHQ= IHBs YXN0 IGNhbg== YWs= b21l dWQ= VGhl IGhpcw== IGRv IGdv IGhhcw== Z2U= J3Q= IFU= cm91 IHNh IGo= IGJ1dA== IHdvcg== IGFsbA== ZWN0 IGs= YW1l IHdpbGw= b2s= IHdoZQ== IHRoZXk= aWRl MDE= ZmY= aWNo cGw= dGhlcg== IHRy Li4= IGludA== aWU= dXJl YWdl IG5l aWFs YXA= aW5l aWNl IG1l IG91dA== YW5z b25l b25n aW9ucw== IHdobw== IEs= IHVw IHRoZWly IGFk IDM= IHVz YXRlZA== b3Vz IG1vcmU= dWU= b2c= IFN0 aW5k aWtl IHNv aW1l cGVy LiI= YmVy aXo= YWN0 IG9uZQ== IHNhaWQ= IC0= YXJl IHlvdXI= Y2M= IFRo IGNs ZXA= YWtl YWJsZQ== aXA= IGNvbnQ= IHdoaWNo aWE= IGlt IGFib3V0 IHdlcmU= dmVyeQ== dWI= IGhhZA== IGVu IGNvbXA= LCI= IElu IHVu IGFn aXJl YWNl YXU= YXJ5 IHdvdWxk YXNz cnk= IOKA Y2w= b29r ZXJl c28= IFY= aWdu aWI= IG9mZg== IHRl dmVu IFk= aWxl b3Nl aXRl b3Jt IDIwMQ== IHJlcw== IG1hbg== IHBlcg== IG90aGVy b3Jk dWx0 IGJlZW4= IGxpa2U= YXNl YW5jZQ== a3M= YXlz b3du ZW5jZQ== IGRpcw== Y3Rpb24= IGFueQ== IGFwcA== IHNw aW50 cmVzcw== YXRpb25z YWls IDQ= aWNhbA== IHRoZW0= IGhlcg== b3VudA== IENo IGFy IGlm IHRoZXJl IHBl IHllYXI= YXY= IG15 IHNvbWU= IHdoZW4= b3VnaA== YWNo IHRoYW4= cnU= b25k aWNr IG92ZXI= dmVs IHF1 Cgo= IHNj cmVhdA== cmVl IEl0 b3VuZA== cG9ydA== IGFsc28= IHBhcnQ= ZnRlcg== IGtu IGJlYw== IHRpbWU= ZW5z IDU= b3BsZQ== IHdoYXQ= IG5v ZHU= bWVy YW5n IG5ldw== LS0tLQ== IGdldA== b3J5 aXRpb24= aW5ncw== IGp1c3Q= IGludG8= IDA= ZW50cw== b3Zl dGU= IHBlb3BsZQ== IHByZQ== IGl0cw== IHJlYw== IHR3 aWFu aXJzdA== YXJr b3Jz IHdvcms= YWRl b2I= IHNoZQ== IG91cg== d24= aW5r bGlj IDE5 IEhl aXNo bmRlcg== YXVzZQ== IGhpbQ== b25z IFs= IHJv Zm9ybQ== aWxk YXRl

// src/ranks/p50k_edit.js
var p50k_edit_default = { "pat_str": "'s|'t|'re|'ve|'m|'ll|'d| ?\\p{L}+| ?\\p{N}+| ?[^\\s\\p{L}\\p{N}]+|\\s+(?!\\S)|\\s+", "special_tokens": { "<|endoftext|>": 50256, "<|fim_prefix|>": 50281, "<|fim_middle|>": 50282, "<|fim_suffix|>": 50283 }, "bpe_ranks": "! 0 IQ== Ig== Iw== JA== JQ== Jg== Jw== KA== KQ== Kg== Kw== LA== LQ== Lg== Lw== MA== MQ== Mg== Mw== NA== NQ== Ng== Nw== OA== OQ== Og== Ow== PA== PQ== Pg== Pw== QA== QQ== Qg== Qw== RA== RQ== Rg== Rw== SA== SQ== Sg== Sw== TA== TQ== Tg== Tw== UA== UQ== Ug== Uw== VA== VQ== Vg== Vw== WA== WQ== Wg== Ww== XA== XQ== Xg== Xw== YA== YQ== Yg== Yw== ZA== ZQ== Zg== Zw== aA== aQ== ag== aw== bA== bQ== bg== bw== cA== cQ== cg== cw== dA== dQ== dg== dw== eA== eQ== eg== ew== fA== fQ== fg== oQ== og== ow== pA== pQ== pg== pw== qA== qQ== qg== qw== rA== rg== rw== sA== sQ== sg== sw== tA== tQ== tg== tw== uA== uQ== ug== uw== vA== vQ== vg== vw== wA== wQ== wg== ww== xA== xQ== xg== xw== yA== yQ== yg== yw== zA== zQ== zg== zw== 0A== 0Q== 0g== 0w== 1A== 1Q== 1g== 1w== 2A== 2Q== 2g== 2w== 3A== 3Q== 3g== 3w== 4A== 4Q== 4g== 4w== 5A== 5Q== 5g== 5w== 6A== 6Q== 6g== 6w== 7A== 7Q== 7g== 7w== 8A== 8Q== 8g== 8w== 9A== 9Q== 9g== 9w== +A== +Q== +g== +w== /A== /Q== /g== /w== AA== AQ== Ag== Aw== BA== BQ== Bg== Bw== CA== CQ== Cg== Cw== DA== DQ== Dg== Dw== EA== EQ== Eg== Ew== FA== FQ== Fg== Fw== GA== GQ== Gg== Gw== HA== HQ== Hg== Hw== IA== fw== gA== gQ== gg== gw== hA== hQ== hg== hw== iA== iQ== ig== iw== jA== jQ== jg== jw== kA== kQ== kg== kw== lA== lQ== lg== lw== mA== mQ== mg== mw== nA== nQ== ng== nw== oA== rQ== IHQ= IGE= aGU= aW4= cmU= b24= IHRoZQ== ZXI= IHM= YXQ= IHc= IG8= ZW4= IGM= aXQ= aXM= YW4= b3I= ZXM= IGI= ZWQ= IGY= aW5n IHA= b3U= IGFu YWw= YXI= IHRv IG0= IG9m IGlu IGQ= IGg= IGFuZA== aWM= YXM= bGU= IHRo aW9u b20= bGw= ZW50 IG4= IGw= c3Q= IHJl dmU= IGU= cm8= bHk= IGJl IGc= IFQ= Y3Q= IFM= aWQ= b3Q= IEk= dXQ= ZXQ= IEE= IGlz IG9u aW0= YW0= b3c= YXk= YWQ= c2U= IHRoYXQ= IEM= aWc= IGZvcg== YWM= IHk= dmVy dXI= IHU= bGQ= IHN0 IE0= J3M= IGhl IGl0 YXRpb24= aXRo aXI= Y2U= IHlvdQ== aWw= IEI= IHdo b2w= IFA= IHdpdGg= IDE= dGVy Y2g= IGFz IHdl ICg= bmQ= aWxs IEQ= aWY= IDI= YWc= ZXJz a2U= ICI= IEg= ZW0= IGNvbg== IFc= IFI= aGVy IHdhcw== IHI= b2Q= IEY= dWw= YXRl IGF0 cmk= cHA= b3Jl IFRoZQ== IHNl dXM= IHBybw== IGhh dW0= IGFyZQ== IGRl YWlu YW5k IG9y aWdo ZXN0 aXN0 YWI= cm9t IE4= dGg= IGNvbQ== IEc= dW4= b3A= MDA= IEw= IG5vdA== ZXNz IGV4 IHY= cmVz IEU= ZXc= aXR5 YW50 IGJ5 ZWw= b3M= b3J0 b2M= cXU= IGZyb20= IGhhdmU= IHN1 aXZl b3VsZA== IHNo IHRoaXM= bnQ= cmE= cGU= aWdodA== YXJ0 bWVudA== IGFs dXN0 ZW5k LS0= YWxs IE8= YWNr IGNo IGxl aWVz cmVk YXJk 4oA= b3V0 IEo= IGFi ZWFy aXY= YWxseQ== b3Vy b3N0 Z2g= cHQ= IHBs YXN0 IGNhbg== YWs= b21l dWQ= VGhl IGhpcw== IGRv IGdv IGhhcw== Z2U= J3Q= IFU= cm91 IHNh IGo= IGJ1dA== IHdvcg== IGFsbA== ZWN0 IGs= YW1l IHdpbGw= b2s= IHdoZQ== IHRoZXk= aWRl MDE= ZmY= aWNo cGw= dGhlcg== IHRy Li4= IGludA== aWU= dXJl YWdl IG5l aWFs YXA= aW5l aWNl IG1l IG91dA== YW5z b25l b25n aW9ucw== IHdobw== IEs= IHVw IHRoZWly IGFk IDM= IHVz YXRlZA== b3Vz IG1vcmU= dWU= b2c= IFN0 aW5k aWtl IHNv aW1l cGVy LiI= YmVy aXo= YWN0 IG9uZQ== IHNhaWQ= IC0= YXJl IHlvdXI= Y2M= IFRo IGNs ZXA= YWtl YWJsZQ== aXA= IGNvbnQ= IHdoaWNo aWE= IGlt IGFib3V0 IHdlcmU= dmVyeQ== dWI= IGhhZA== IGVu IGNvbXA= LCI= IElu IHVu IGFn aXJl YWNl YXU= YXJ5 IHdvdWxk YXNz cnk= IOKA Y2w= b29r ZXJl c28= IFY= aWdu aWI= IG9mZg== IHRl dmVu IFk= aWxl b3Nl aXRl b3Jt IDIwMQ== IHJlcw== IG1hbg== IHBlcg== IG90aGVy b3Jk dWx0 IGJlZW4= IGxpa2U= YXNl YW5jZQ== a3M= YXlz b3du ZW5jZQ== IGRpcw== Y3Rpb24= IGFueQ== IGFwcA== IHNw aW50 cmVzcw== YXRpb25z YWls IDQ= aWNhbA== IHRoZW0= IGhlcg== b3VudA== IENo IGFy IGlm IHRoZXJl IHBl IHllYXI= YXY= IG15 IHNvbWU= IHdoZW4= b3VnaA== YWNo IHRoYW4= cnU= b25k aWNr IG92ZXI= dmVs IHF1 Cgo= IHNj cmVhdA== cmVl IEl0 b3VuZA== cG9ydA== IGFsc28= IHBhcnQ= ZnRlcg== IGtu IGJlYw== IHRpbWU= ZW5z IDU= b3BsZQ== IHdoYXQ= IG5v ZHU= bWVy YW5n IG5ldw== LS0tLQ== IGdldA== b3J5 aXRpb24= aW5ncw== IGp1c3Q= IGludG8= IDA= ZW50cw== b3Zl dGU= IHBlb3BsZQ== IHByZQ== IGl0cw== IHJlYw== IHR3 aWFu aXJzdA== YXJr b3Jz IHdvcms= YWRl b2I= IHNoZQ== IG91cg== d24= aW5r bGlj IDE5 IEhl aXNo bmRlcg== YXV

// src/ranks/r50k_base.js
var r50k_base_default = { "explicit_n_vocab": 50257, "pat_str": "'s|'t|'re|'ve|'m|'ll|'d| ?\\p{L}+| ?\\p{N}+| ?[^\\s\\p{L}\\p{N}]+|\\s+(?!\\S)|\\s+", "special_tokens": { "<|endoftext|>": 50256 }, "bpe_ranks": "! 0 IQ== Ig== Iw== JA== JQ== Jg== Jw== KA== KQ== Kg== Kw== LA== LQ== Lg== Lw== MA== MQ== Mg== Mw== NA== NQ== Ng== Nw== OA== OQ== Og== Ow== PA== PQ== Pg== Pw== QA== QQ== Qg== Qw== RA== RQ== Rg== Rw== SA== SQ== Sg== Sw== TA== TQ== Tg== Tw== UA== UQ== Ug== Uw== VA== VQ== Vg== Vw== WA== WQ== Wg== Ww== XA== XQ== Xg== Xw== YA== YQ== Yg== Yw== ZA== ZQ== Zg== Zw== aA== aQ== ag== aw== bA== bQ== bg== bw== cA== cQ== cg== cw== dA== dQ== dg== dw== eA== eQ== eg== ew== fA== fQ== fg== oQ== og== ow== pA== pQ== pg== pw== qA== qQ== qg== qw== rA== rg== rw== sA== sQ== sg== sw== tA== tQ== tg== tw== uA== uQ== ug== uw== vA== vQ== vg== vw== wA== wQ== wg== ww== xA== xQ== xg== xw== yA== yQ== yg== yw== zA== zQ== zg== zw== 0A== 0Q== 0g== 0w== 1A== 1Q== 1g== 1w== 2A== 2Q== 2g== 2w== 3A== 3Q== 3g== 3w== 4A== 4Q== 4g== 4w== 5A== 5Q== 5g== 5w== 6A== 6Q== 6g== 6w== 7A== 7Q== 7g== 7w== 8A== 8Q== 8g== 8w== 9A== 9Q== 9g== 9w== +A== +Q== +g== +w== /A== /Q== /g== /w== AA== AQ== Ag== Aw== BA== BQ== Bg== Bw== CA== CQ== Cg== Cw== DA== DQ== Dg== Dw== EA== EQ== Eg== Ew== FA== FQ== Fg== Fw== GA== GQ== Gg== Gw== HA== HQ== Hg== Hw== IA== fw== gA== gQ== gg== gw== hA== hQ== hg== hw== iA== iQ== ig== iw== jA== jQ== jg== jw== kA== kQ== kg== kw== lA== lQ== lg== lw== mA== mQ== mg== mw== nA== nQ== ng== nw== oA== rQ== IHQ= IGE= aGU= aW4= cmU= b24= IHRoZQ== ZXI= IHM= YXQ= IHc= IG8= ZW4= IGM= aXQ= aXM= YW4= b3I= ZXM= IGI= ZWQ= IGY= aW5n IHA= b3U= IGFu YWw= YXI= IHRv IG0= IG9m IGlu IGQ= IGg= IGFuZA== aWM= YXM= bGU= IHRo aW9u b20= bGw= ZW50 IG4= IGw= c3Q= IHJl dmU= IGU= cm8= bHk= IGJl IGc= IFQ= Y3Q= IFM= aWQ= b3Q= IEk= dXQ= ZXQ= IEE= IGlz IG9u aW0= YW0= b3c= YXk= YWQ= c2U= IHRoYXQ= IEM= aWc= IGZvcg== YWM= IHk= dmVy dXI= IHU= bGQ= IHN0 IE0= J3M= IGhl IGl0 YXRpb24= aXRo aXI= Y2U= IHlvdQ== aWw= IEI= IHdo b2w= IFA= IHdpdGg= IDE= dGVy Y2g= IGFz IHdl ICg= bmQ= aWxs IEQ= aWY= IDI= YWc= ZXJz a2U= ICI= IEg= ZW0= IGNvbg== IFc= IFI= aGVy IHdhcw== IHI= b2Q= IEY= dWw= YXRl IGF0 cmk= cHA= b3Jl IFRoZQ== IHNl dXM= IHBybw== IGhh dW0= IGFyZQ== IGRl YWlu YW5k IG9y aWdo ZXN0 aXN0 YWI= cm9t IE4= dGg= IGNvbQ== IEc= dW4= b3A= MDA= IEw= IG5vdA== ZXNz IGV4 IHY= cmVz IEU= ZXc= aXR5 YW50 IGJ5 ZWw= b3M= b3J0 b2M= cXU= IGZyb20= IGhhdmU= IHN1 aXZl b3VsZA== IHNo IHRoaXM= bnQ= cmE= cGU= aWdodA== YXJ0 bWVudA== IGFs dXN0 ZW5k LS0= YWxs IE8= YWNr IGNo IGxl aWVz cmVk YXJk 4oA= b3V0 IEo= IGFi ZWFy aXY= YWxseQ== b3Vy b3N0 Z2g= cHQ= IHBs YXN0 IGNhbg== YWs= b21l dWQ= VGhl IGhpcw== IGRv IGdv IGhhcw== Z2U= J3Q= IFU= cm91 IHNh IGo= IGJ1dA== IHdvcg== IGFsbA== ZWN0 IGs= YW1l IHdpbGw= b2s= IHdoZQ== IHRoZXk= aWRl MDE= ZmY= aWNo cGw= dGhlcg== IHRy Li4= IGludA== aWU= dXJl YWdl IG5l aWFs YXA= aW5l aWNl IG1l IG91dA== YW5z b25l b25n aW9ucw== IHdobw== IEs= IHVw IHRoZWly IGFk IDM= IHVz YXRlZA== b3Vz IG1vcmU= dWU= b2c= IFN0 aW5k aWtl IHNv aW1l cGVy LiI= YmVy aXo= YWN0 IG9uZQ== IHNhaWQ= IC0= YXJl IHlvdXI= Y2M= IFRo IGNs ZXA= YWtl YWJsZQ== aXA= IGNvbnQ= IHdoaWNo aWE= IGlt IGFib3V0 IHdlcmU= dmVyeQ== dWI= IGhhZA== IGVu IGNvbXA= LCI= IElu IHVu IGFn aXJl YWNl YXU= YXJ5 IHdvdWxk YXNz cnk= IOKA Y2w= b29r ZXJl c28= IFY= aWdu aWI= IG9mZg== IHRl dmVu IFk= aWxl b3Nl aXRl b3Jt IDIwMQ== IHJlcw== IG1hbg== IHBlcg== IG90aGVy b3Jk dWx0 IGJlZW4= IGxpa2U= YXNl YW5jZQ== a3M= YXlz b3du ZW5jZQ== IGRpcw== Y3Rpb24= IGFueQ== IGFwcA== IHNw aW50 cmVzcw== YXRpb25z YWls IDQ= aWNhbA== IHRoZW0= IGhlcg== b3VudA== IENo IGFy IGlm IHRoZXJl IHBl IHllYXI= YXY= IG15 IHNvbWU= IHdoZW4= b3VnaA== YWNo IHRoYW4= cnU= b25k aWNr IG92ZXI= dmVs IHF1 Cgo= IHNj cmVhdA== cmVl IEl0 b3VuZA== cG9ydA== IGFsc28= IHBhcnQ= ZnRlcg== IGtu IGJlYw== IHRpbWU= ZW5z IDU= b3BsZQ== IHdoYXQ= IG5v ZHU= bWVy YW5n IG5ldw== LS0tLQ== IGdldA== b3J5 aXRpb24= aW5ncw== IGp1c3Q= IGludG8= IDA= ZW50cw== b3Zl dGU= IHBlb3BsZQ== IHByZQ== IGl0cw== IHJlYw== IHR3 aWFu aXJzdA== YXJr b3Jz IHdvcms= YWRl b2I= IHNoZQ== IG91cg== d24= aW5r bGlj IDE5 IEhl aXNo bmRlcg== YXVzZQ== IGhpbQ== b25z IFs= IHJv Zm9ybQ== aWxk YXRl

// src/ranks/cl100k_base.js
var cl100k_base_default = { "pat_str": "('s|'S|'t|'T|'re|'rE|'Re|'RE|'ve|'vE|'Ve|'VE|'m|'M|'ll|'lL|'Ll|'LL|'d|'D)|[^\\r\\n\\p{L}\\p{N}]?\\p{L}+|\\p{N}{1,3}| ?[^\\s\\p{L}\\p{N}]+[\\r\\n]*|\\s*[\\r\\n]+|\\s+(?!\\S)|\\s+", "special_tokens": { "<|endoftext|>": 100257, "<|fim_prefix|>": 100258, "<|fim_middle|>": 100259, "<|fim_suffix|>": 100260, "<|endofprompt|>": 100276 }, "bpe_ranks": "! 0 IQ== Ig== Iw== JA== JQ== Jg== Jw== KA== KQ== Kg== Kw== LA== LQ== Lg== Lw== MA== MQ== Mg== Mw== NA== NQ== Ng== Nw== OA== OQ== Og== Ow== PA== PQ== Pg== Pw== QA== QQ== Qg== Qw== RA== RQ== Rg== Rw== SA== SQ== Sg== Sw== TA== TQ== Tg== Tw== UA== UQ== Ug== Uw== VA== VQ== Vg== Vw== WA== WQ== Wg== Ww== XA== XQ== Xg== Xw== YA== YQ== Yg== Yw== ZA== ZQ== Zg== Zw== aA== aQ== ag== aw== bA== bQ== bg== bw== cA== cQ== cg== cw== dA== dQ== dg== dw== eA== eQ== eg== ew== fA== fQ== fg== oQ== og== ow== pA== pQ== pg== pw== qA== qQ== qg== qw== rA== rg== rw== sA== sQ== sg== sw== tA== tQ== tg== tw== uA== uQ== ug== uw== vA== vQ== vg== vw== wA== wQ== wg== ww== xA== xQ== xg== xw== yA== yQ== yg== yw== zA== zQ== zg== zw== 0A== 0Q== 0g== 0w== 1A== 1Q== 1g== 1w== 2A== 2Q== 2g== 2w== 3A== 3Q== 3g== 3w== 4A== 4Q== 4g== 4w== 5A== 5Q== 5g== 5w== 6A== 6Q== 6g== 6w== 7A== 7Q== 7g== 7w== 8A== 8Q== 8g== 8w== 9A== 9Q== 9g== 9w== +A== +Q== +g== +w== /A== /Q== /g== /w== AA== AQ== Ag== Aw== BA== BQ== Bg== Bw== CA== CQ== Cg== Cw== DA== DQ== Dg== Dw== EA== EQ== Eg== Ew== FA== FQ== Fg== Fw== GA== GQ== Gg== Gw== HA== HQ== Hg== Hw== IA== fw== gA== gQ== gg== gw== hA== hQ== hg== hw== iA== iQ== ig== iw== jA== jQ== jg== jw== kA== kQ== kg== kw== lA== lQ== lg== lw== mA== mQ== mg== mw== nA== nQ== ng== nw== oA== rQ== ICA= ICAgIA== aW4= IHQ= ICAgICAgICA= ZXI= ICAg b24= IGE= cmU= YXQ= c3Q= ZW4= b3I= IHRo Cgo= IGM= bGU= IHM= aXQ= YW4= YXI= YWw= IHRoZQ== Owo= IHA= IGY= b3U= ID0= aXM= ICAgICAgIA== aW5n ZXM= IHc= aW9u ZWQ= aWM= IGI= IGQ= ZXQ= IG0= IG8= CQk= cm8= YXM= ZWw= Y3Q= bmQ= IGlu IGg= ZW50 aWQ= IG4= YW0= ICAgICAgICAgICA= IHRv IHJl LS0= IHs= IG9m b20= KTsK aW0= DQo= ICg= aWw= Ly8= IGFuZA== dXI= c2U= IGw= ZXg= IFM= YWQ= ICI= Y2g= dXQ= aWY= Kio= IH0= ZW0= b2w= ICAgICAgICAgICAgICAgIA== dGg= KQo= IHsK IGc= aWc= aXY= LAo= Y2U= b2Q= IHY= YXRl IFQ= YWc= YXk= ICo= b3Q= dXM= IEM= IHN0 IEk= dW4= dWw= dWU= IEE= b3c= ICc= ZXc= IDw= YXRpb24= KCk= IGZvcg== YWI= b3J0 dW0= YW1l IGlz cGU= dHI= Y2s= 4oA= IHk= aXN0 LS0tLQ== LgoK aGU= IGU= bG8= IE0= IGJl ZXJz IG9u IGNvbg== YXA= dWI= IFA= ICAgICAgICAgICAgICAg YXNz aW50 Pgo= bHk= dXJu ICQ= OwoK YXY= cG9ydA== aXI= LT4= bnQ= Y3Rpb24= ZW5k IGRl MDA= aXRo b3V0 dHVybg== b3Vy ICAgICA= bGlj cmVz cHQ= PT0= IHRoaXM= IHdo IGlm IEQ= dmVy YWdl IEI= aHQ= ZXh0 PSI= IHRoYXQ= KioqKg== IFI= IGl0 ZXNz IEY= IHI= b3M= YW5k IGFz ZWN0 a2U= cm9t IC8v Y29u IEw= KCI= cXU= bGFzcw== IHdpdGg= aXo= ZGU= IE4= IGFs b3A= dXA= Z2V0 IH0K aWxl IGFu YXRh b3Jl cmk= IHBybw== Ow0K CQkJCQ== dGVy YWlu IFc= IEU= IGNvbQ== IHJldHVybg== YXJ0 IEg= YWNr aW1wb3J0 dWJsaWM= IG9y ZXN0 bWVudA== IEc= YWJsZQ== IC0= aW5l aWxs aW5k ZXJl Ojo= aXR5 ICs= IHRy ZWxm aWdodA== KCc= b3Jt dWx0 c3Ry Li4= Iiw= IHlvdQ== eXBl cGw= IG5ldw== IGo= ICAgICAgICAgICAgICAgICAgIA== IGZyb20= IGV4 IE8= MjA= bGQ= IFs= b2M= Ogo= IHNl IGxl LS0tLS0tLS0= LnM= ewo= Jyw= YW50 IGF0 YXNl LmM= IGNo PC8= YXZl YW5n IGFyZQ== IGludA== 4oCZ X3Q= ZXJ0 aWFs YWN0 fQo= aXZl b2Rl b3N0 IGNsYXNz IG5vdA== b2c= b3Jk YWx1ZQ== YWxs ZmY= KCk7Cg== b250 aW1l YXJl IFU= IHBy IDo= aWVz aXpl dXJl IGJ5 aXJl IH0KCg== LnA= IHNo aWNl YXN0 cHRpb24= dHJpbmc= b2s= X18= Y2w= IyM= IGhl YXJk KS4= IEA= aWV3 CQkJ IHdhcw== aXA= dGhpcw== IHU= IFRoZQ== aWRl YWNl aWI= YWM= cm91 IHdl amVjdA== IHB1YmxpYw== YWs= dmU= YXRo b2lk ID0+ dXN0 cXVl IHJlcw== KSk= J3M= IGs= YW5z eXN0 dW5jdGlvbg== KioqKioqKio= IGk= IHVz cHA= MTA= b25l YWls PT09PQ== bmFtZQ== IHN0cg== IC8= ICY= YWNo ZGl2 eXN0ZW0= ZWxs IGhhdmU= ZXJy b3VsZA== dWxs cG9u IEo= X3A= ID09 aWdu U3Q= Lgo= IHBs KTsKCg== Zm9ybQ== cHV0 b3VudA== fQoK ZGQ= aXRl IGdldA== cnI= b21l IOKA YXJhbQ== Y2M= ICov RVI= SW4= bGVz X3M= b25n aWU= IGNhbg== IFY= ZXJ2 cHI= IHVu cm93 YmVy IGRv bGw= IGVs IHNlbGY= YXRlZA== YXJ5 IC4= J10= dWQ= IGVu IFRo ICAgICAgICAgICAgICAgIC

// src/index.ts
function getEncoding(encoding, extendSpecialTokens) {
  switch (encoding) {
    case "gpt2":
      return new Tiktoken(gpt2_default, extendSpecialTokens);
    case "r50k_base":
      return new Tiktoken(r50k_base_default, extendSpecialTokens);
    case "p50k_base":
      return new Tiktoken(p50k_base_default, extendSpecialTokens);
    case "p50k_edit":
      return new Tiktoken(p50k_edit_default, extendSpecialTokens);
    case "cl100k_base":
      return new Tiktoken(cl100k_base_default, extendSpecialTokens);
    default:
      never(encoding);
      throw new Error("Unknown encoding");
  }
}
function encodingForModel(model, extendSpecialTokens) {
  return getEncoding(getEncodingNameForModel(model), extendSpecialTokens);
}

export { encodingForModel, getEncoding };