-
Notifications
You must be signed in to change notification settings - Fork 0
/
fix_unicode_filenames.py
1639 lines (1299 loc) · 88.6 KB
/
fix_unicode_filenames.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
"""
*** SAVE CODE CHANGES TO A .SAVETEXT FILE FIRST AND CHECK ENCODING BEFORE OVERWRITING!!! IT IS VERY EASY TO CORRUPT THIS FILE!!! TOO EASY!!! ***
*** SAVE CODE CHANGES TO A .SAVETEXT FILE FIRST AND CHECK ENCODING BEFORE OVERWRITING!!! IT IS VERY EASY TO CORRUPT THIS FILE!!! TOO EASY!!! ***
*** SAVE CODE CHANGES TO A .SAVETEXT FILE FIRST AND CHECK ENCODING BEFORE OVERWRITING!!! IT IS VERY EASY TO CORRUPT THIS FILE!!! TOO EASY!!! ***
*** SAVE CODE CHANGES TO A .SAVETEXT FILE FIRST AND CHECK ENCODING BEFORE OVERWRITING!!! IT IS VERY EASY TO CORRUPT THIS FILE!!! TOO EASY!!! ***
*** SAVE CODE CHANGES TO A .SAVETEXT FILE FIRST AND CHECK ENCODING BEFORE OVERWRITING!!! IT IS VERY EASY TO CORRUPT THIS FILE!!! TOO EASY!!! ***
*** SAVE CODE CHANGES TO A .SAVETEXT FILE FIRST AND CHECK ENCODING BEFORE OVERWRITING!!! IT IS VERY EASY TO CORRUPT THIS FILE!!! TOO EASY!!! ***
*** SAVE CODE CHANGES TO A .SAVETEXT FILE FIRST AND CHECK ENCODING BEFORE OVERWRITING!!! IT IS VERY EASY TO CORRUPT THIS FILE!!! TOO EASY!!! ***
*** SAVE CODE CHANGES TO A .SAVETEXT FILE FIRST AND CHECK ENCODING BEFORE OVERWRITING!!! IT IS VERY EASY TO CORRUPT THIS FILE!!! TOO EASY!!! ***
*** SAVE CODE CHANGES TO A .SAVETEXT FILE FIRST AND CHECK ENCODING BEFORE OVERWRITING!!! IT IS VERY EASY TO CORRUPT THIS FILE!!! TOO EASY!!! ***
*** SAVE CODE CHANGES TO A .SAVETEXT FILE FIRST AND CHECK ENCODING BEFORE OVERWRITING!!! IT IS VERY EASY TO CORRUPT THIS FILE!!! TOO EASY!!! ***
*** SAVE CODE CHANGES TO A .SAVETEXT FILE FIRST AND CHECK ENCODING BEFORE OVERWRITING!!! IT IS VERY EASY TO CORRUPT THIS FILE!!! TOO EASY!!! ***
*** SAVE CODE CHANGES TO A .SAVETEXT FILE FIRST AND CHECK ENCODING BEFORE OVERWRITING!!! IT IS VERY EASY TO CORRUPT THIS FILE!!! TOO EASY!!! ***
*** SAVE CODE CHANGES TO A .SAVETEXT FILE FIRST AND CHECK ENCODING BEFORE OVERWRITING!!! IT IS VERY EASY TO CORRUPT THIS FILE!!! TOO EASY!!! ***
*** SAVE CODE CHANGES TO A .SAVETEXT FILE FIRST AND CHECK ENCODING BEFORE OVERWRITING!!! IT IS VERY EASY TO CORRUPT THIS FILE!!! TOO EASY!!! ***
*** SAVE CODE CHANGES TO A .SAVETEXT FILE FIRST AND CHECK ENCODING BEFORE OVERWRITING!!! IT IS VERY EASY TO CORRUPT THIS FILE!!! TOO EASY!!! ***
*** SAVE CODE CHANGES TO A .SAVETEXT FILE FIRST AND CHECK ENCODING BEFORE OVERWRITING!!! IT IS VERY EASY TO CORRUPT THIS FILE!!! TOO EASY!!! ***
*** SAVE CODE CHANGES TO A .SAVETEXT FILE FIRST AND CHECK ENCODING BEFORE OVERWRITING!!! IT IS VERY EASY TO CORRUPT THIS FILE!!! TOO EASY!!! ***
*** SAVE CODE CHANGES TO A .SAVETEXT FILE FIRST AND CHECK ENCODING BEFORE OVERWRITING!!! IT IS VERY EASY TO CORRUPT THIS FILE!!! TOO EASY!!! ***
*** SAVE CODE CHANGES TO A .SAVETEXT FILE FIRST AND CHECK ENCODING BEFORE OVERWRITING!!! IT IS VERY EASY TO CORRUPT THIS FILE!!! TOO EASY!!! ***
*** SAVE CODE CHANGES TO A .SAVETEXT FILE FIRST AND CHECK ENCODING BEFORE OVERWRITING!!! IT IS VERY EASY TO CORRUPT THIS FILE!!! TOO EASY!!! ***
*** SAVE CODE CHANGES TO A .SAVETEXT FILE FIRST AND CHECK ENCODING BEFORE OVERWRITING!!! IT IS VERY EASY TO CORRUPT THIS FILE!!! TOO EASY!!! ***
*** SAVE CODE CHANGES TO A .SAVETEXT FILE FIRST AND CHECK ENCODING BEFORE OVERWRITING!!! IT IS VERY EASY TO CORRUPT THIS FILE!!! TOO EASY!!! ***
*** SAVE CODE CHANGES TO A .SAVETEXT FILE FIRST AND CHECK ENCODING BEFORE OVERWRITING!!! IT IS VERY EASY TO CORRUPT THIS FILE!!! TOO EASY!!! ***
*** SAVE CODE CHANGES TO A .SAVETEXT FILE FIRST AND CHECK ENCODING BEFORE OVERWRITING!!! IT IS VERY EASY TO CORRUPT THIS FILE!!! TOO EASY!!! ***
*** SAVE CODE CHANGES TO A .SAVETEXT FILE FIRST AND CHECK ENCODING BEFORE OVERWRITING!!! IT IS VERY EASY TO CORRUPT THIS FILE!!! TOO EASY!!! ***
*** SAVE CODE CHANGES TO A .SAVETEXT FILE FIRST AND CHECK ENCODING BEFORE OVERWRITING!!! IT IS VERY EASY TO CORRUPT THIS FILE!!! TOO EASY!!! ***
*** SAVE CODE CHANGES TO A .SAVETEXT FILE FIRST AND CHECK ENCODING BEFORE OVERWRITING!!! IT IS VERY EASY TO CORRUPT THIS FILE!!! TOO EASY!!! ***
*** SAVE CODE CHANGES TO A .SAVETEXT FILE FIRST AND CHECK ENCODING BEFORE OVERWRITING!!! IT IS VERY EASY TO CORRUPT THIS FILE!!! TOO EASY!!! ***
*** SAVE CODE CHANGES TO A .SAVETEXT FILE FIRST AND CHECK ENCODING BEFORE OVERWRITING!!! IT IS VERY EASY TO CORRUPT THIS FILE!!! TOO EASY!!! ***
*** SAVE CODE CHANGES TO A .SAVETEXT FILE FIRST AND CHECK ENCODING BEFORE OVERWRITING!!! IT IS VERY EASY TO CORRUPT THIS FILE!!! TOO EASY!!! ***
*** SAVE CODE CHANGES TO A .SAVETEXT FILE FIRST AND CHECK ENCODING BEFORE OVERWRITING!!! IT IS VERY EASY TO CORRUPT THIS FILE!!! TOO EASY!!! ***
*** SAVE CODE CHANGES TO A .SAVETEXT FILE FIRST AND CHECK ENCODING BEFORE OVERWRITING!!! IT IS VERY EASY TO CORRUPT THIS FILE!!! TOO EASY!!! ***
*** SAVE CODE CHANGES TO A .SAVETEXT FILE FIRST AND CHECK ENCODING BEFORE OVERWRITING!!! IT IS VERY EASY TO CORRUPT THIS FILE!!! TOO EASY!!! ***
*** SAVE CODE CHANGES TO A .SAVETEXT FILE FIRST AND CHECK ENCODING BEFORE OVERWRITING!!! IT IS VERY EASY TO CORRUPT THIS FILE!!! TOO EASY!!! ***
*** SAVE CODE CHANGES TO A .SAVETEXT FILE FIRST AND CHECK ENCODING BEFORE OVERWRITING!!! IT IS VERY EASY TO CORRUPT THIS FILE!!! TOO EASY!!! ***
*** SAVE CODE CHANGES TO A .SAVETEXT FILE FIRST AND CHECK ENCODING BEFORE OVERWRITING!!! IT IS VERY EASY TO CORRUPT THIS FILE!!! TOO EASY!!! ***
*** SAVE CODE CHANGES TO A .SAVETEXT FILE FIRST AND CHECK ENCODING BEFORE OVERWRITING!!! IT IS VERY EASY TO CORRUPT THIS FILE!!! TOO EASY!!! ***
*** SAVE CODE CHANGES TO A .SAVETEXT FILE FIRST AND CHECK ENCODING BEFORE OVERWRITING!!! IT IS VERY EASY TO CORRUPT THIS FILE!!! TOO EASY!!! ***
Coverts Unicode/non-ASCII filenames into ASCII filenames -- "Romanizing-Plus"
USAGE:
SETUP: To suppress user prompting: set AUTOMATIC_UNICODE_CLEANING=1
RECURSIVE: add "/s" to the end to recurse folders [in filemode only, obviously]
MODE 1: No arguments : Run with no arguments to cleanse everything in your existing folder of unicode characters
: "auto" argument : ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ Do this, but suppress confirmation prompts
MODE 2: "file <arguments>": Use "file" as your first argument to cleanse the rest of the command line of unicode, as if it were a windows filename
MODE 3: "string <arguments>": Use "string" as your first argument to cleanse the rest of the command line of unicode, without restricting to only-valid-in-windows-fiklenames
MODE 4: "test" : to convert the internal testing string
MODE 5: "script" : experimental
EXAMPLE PROGRAMMATIC USAGE:
import fixUnicodeFilenames
a_string_without_unicode = fixUnicodeFilenames.convert_a_string (original_stringval_with_unicode,silent=False)
filename_without_unicode = fixUnicodeFilenames.convert_a_filename(original_file_name_with_unicode,silent_if_unchanged=True,silent_if_changed=True)
#silent=suppresses all output no matter what
Uses Polyglot library to attempt a language-agnostic translation, which can easliy fail
Then several internal custom mapping tables for phonetically romanizing characters & emojis
Then several lingual libraries for romanizing individual characters for some "weirder alphabet" languages
Then an emoji library for converting unconverted emojis
"""
#pylint: disable=C0103,C0413,W0719,R1726
import os
os.system("") #necessary bugfix, believe it or not #GOAT but let's try taking it out to challenge ourselves and maybe speedup startup time
os.environ['PYTHAINLP_ZONEINFO_PACKAGE'] = 'tzdata' #necessary bugfix, believe it or not
import sys ; sys.setrecursionlimit(sys.getrecursionlimit() * 5) #recursionlimit came up during EXE-build attempts
import shutil
import msvcrt
import builtins
#import unidecode #pip install Unidecode==1.2.0 - for the right one - capitalizing the U (or not) is (or isn't) important. this package sucks.
import unicodedata
from unidecode import unidecode
from colorama import Fore, Back, Style, just_fix_windows_console
#init()
just_fix_windows_console()
import clairecjs_utils as claire
import fix_unicode_filenames_every_char as everychar
original_print = print #Store the original print function before any potential overriding
############################ RUNTIME CONFIGURATION ############################
VALID_MODES = ["string", "file", "test", "script"]
INVALID_WINDOWS_FILENAME_CHARACTERS = r'<>:"/\|?*'
RECURSE=False #Whether we are in recursive mode or not
###############################################################################
########################## DEVELOPMENT CONFIGURATION ############################
DIE_ON_UNDECODEABLE_UNICODE_CHARACTER = True
DRY_RUN = False
#################################################################################
################################## DEBUG CONFIGURATION ################################################
DEBUG_MOST_CHARS = False #controls several debugs below
DEBUG_ALL_CHARS = False #controls several debugs below
DEBUG_ANNOUNCE_FILENAMES=True
DEBUG_MODE_ARGV=False
DEBUG_LANG_DETECT=False
DEBUG_POLYGLOT=False
DEBUG_CHAR = bool(False or DEBUG_ALL_CHARS or DEBUG_MOST_CHARS)
DEBUG_UNIDECODECHAR = bool(False or DEBUG_ALL_CHARS or DEBUG_MOST_CHARS)
DEBUG_UNIDECODECHAR_TRANSLATECHAR = bool(False
or DEBUG_ALL_CHARS) # super verbose
DEBUG_INTERNAL_TESTING=False
#######################################################################################################
###################################### TESTING ######################################
## CREATE A GOOD TESTING STRING:
#
# This string includes:
#
# ASCII text ("Hello, world!")
# Chinese text ("ä½ å¥½ï¼Œä¸–ç•Œï¼")
# Japanese text ("ã“ã‚“ã«ã¡ã¯ã€ä¸–ç•Œï¼")
# Korean text ("안녕하세요, 세계!")
# Russian text ("Привет, мир!")
# Greek text ("ΚαλημÎÏα κόσμε!")
# Emoji ("👋ðŸŒ")
# Special symbols and numbers ("âµâ„“©®½¼¾⅓⅔⅛⅜â…â…žâ°Â¹Â²Â³â´âµâ¶â·â¸â¹")
# Mathematical Alphanumeric Symbols (ð‘¨ð’ƒð’„ð’…ð’†ð’‡ð’ˆð’‰ð’Šð’‹ð’Œð’ð’Žð’ð’ð’‘ð’’ð’“ð’”ð’•ð’–ð’—ð’˜ð’™ð’šð’› ð‘¨ð‘©ð‘ªð‘«ð‘¬ð‘ð‘®ð‘¯ð‘°ð‘±ð‘²ð‘³ð‘´ð‘µð‘¶ð‘·ð‘¸ð‘¹ð‘ºð‘»ð‘¼ð‘½ð‘¾ð‘¿ð’€ð’)
# Hebrew and Arabic scripts ("×ָלֶף بÙيت")
# Various letters and symbols from different scripts (DŽDždžLJLjljNJNjnj ỒợỔộ ỖỗỘộ ứỤựỦỠỨữ Ừá»á»¬á»« ỮỠỰự)
massive_testing_string_backup = "½¼¾⅓⅔⅛⅜â…â…ž"
massive_testing_string = """
Hello, world! ä½ å¥½ï¼Œä¸–ç•Œï¼ã“ã‚“ã«ã¡ã¯ã€ä¸–ç•Œï¼ì•ˆë…•í•˜ì„¸ìš”, 세계! Привет, мир! ΚαλημÎÏα κόσμε!
HAND="👋",WORLD="ðŸŒ" âµâ„“ COPYRIGHT="©",RESTRICT="®"
½¼¾⅓⅔⅛⅜â…â…žâ°Â¹Â²Â³â´âµâ¶â·â¸â¹
ð‘¨ð’ƒð’„ð’…ð’†ð’‡ð’ˆð’‰ð’Šð’‹ð’Œð’ð’Žð’ð’ð’‘ð’’ð’“ð’”ð’•ð’–ð’—ð’˜ð’™ð’šð’› ð‘¨ð‘©ð‘ªð‘«ð‘¬ð‘ð‘®ð‘¯ð‘°ð‘±ð‘²ð‘³ð‘´ð‘µð‘¶ð‘·ð‘¸ð‘¹ð‘ºð‘»ð‘¼ð‘½ð‘¾ð‘¿ð’€ð’
×ָלֶף بÙيت DŽDždžLJLjljNJNjnj ỒợỔộ ỖỗỘộ ứỤựỦỠỨữ Ừá»á»¬á»« ỮỠỰự
"""
#####################################################################################
def print_error(*args, called_from_primt=False, **kwargs): #pylint: disable=W0613
if not called_from_primt: raise Exception("A print statement was used in the code. Use primt instead, because we want everything to go to our logfile") #pylint: disable=W0719
def primt(*args, **kwargs): #custom_print "prim print" function to print, prim and proper, to screen & logfile at the same time
global LOGFILE
new_args = []
for arg in args:
if isinstance(arg, str):
new_arg = unidecode(arg)
new_args.append(new_arg)
else:
new_args.append(arg)
output = " ".join(map(str, new_args))
original_print(output, **kwargs) # Call the original print function that we saved before
with open("fix-unicode-filenames.log", "a", encoding='utf-8') as log_file:
#log_file.write(f"{strip_ansi_codes(output)}\n")
log_file.write(f"{output}\n")
def convert_to_ascii_filename_chracters(filename,mode):
"""Translates a string (in our case, a filename) to its ASCII/roman equivalent
(1) First, an amazing multi-language language-agnostic full translation library called polyglot is used
to interpret the entire filename/string at a high ("smart") level to see if a language is detected,
and then to make language-specific conversions to our ASCII/roman equivalent characters.
But it throws an exception if a specific language is not detected, and it's also hard to install,
so the entire thing is wrapped around an exception that just throws the original text if anything goes wrong.
Also, polyglot will omit characters sometimes, so we do not want a null string
(2) Then, each character is processed at a per-chracter level, checking its unicode range to see if it's a language,
and then passing through either a language library or a phoenetic mapping table, to translate the chracters
back to ASCII/roman.
"""
global DEBUG, DEBUG_POLYGLOT
string_romanized_with_polyglot = polyglot_language_agnostic_romanize(filename)
if DEBUG_POLYGLOT:
primt(f'DEBUG: string_romanized_with_polyglot({filename}) is "{string_romanized_with_polyglot}"')
return ''.join(translate_character_with_language_libraries(char,mode,filename=filename) for char in string_romanized_with_polyglot)
#pylint: disable=C0415 #don't nag me about lazy-loading the libraries, pylint!
def polyglot_language_agnostic_romanize(text):
"""Return translated text, but fail very gracefully and transparently if there are any exceptions"""
global DEBUG, DEBUG_LANG_DETECT
try:
import logging
if not DEBUG_LANG_DETECT: logging.getLogger('polyglot').setLevel(logging.ERROR) #Disable logging messages from Polyglot unless in debug mode
from polyglot.detect import Detector
from polyglot.transliteration import Transliterator
detector = Detector(text)
if DEBUG_LANG_DETECT: primt(f"* Detector: {str(detector)}")
source_lang = detector.language.code
transliterator = Transliterator(source_lang=source_lang, target_lang="en")
return transliterator.transliterate(text)
except Exception: #pylint: disable=W0718
return text
def get_unicode_hex(character):
if character == "": return "0" #just fill in a dummy value
return "\\u" + hex(ord(character))[2:].zfill(4) #thank you ChatGPT
def translate_one_or_more_chars_with_custom_character_mapping(chars, mode): #pylint: disable=R0912
"""
Returns characters (after mapping), and done (boolean) which i used for flow control in the outer scope the first time it's called, but ignored the second time it's called
"""
global DEBUG_UNIDECODECHAR_TRANSLATECHAR, VALID_MODES
if mode not in VALID_MODES:
primt(f"{Fore.RED}FATAL TRANSLATE ERROR: translate_one_or_more_chars_with_custom_character_mapping called with invalid mode of {mode} which is not in {VALID_MODES}")
sys.exit(666)
translated_chars = []
done = False
code2 = "" # unicode code without the \ before it
for char in chars: # If it's not in our custom mapping, we basically pass through without doing anything
code, code2, code3 = "", "", ""
if DEBUG_UNIDECODECHAR_TRANSLATECHAR:
code = get_unicode_hex(char)
code2 = "code " + str(get_unicode_hex(char)).replace("\\","")
primt(f"\t{Fore.CYAN}translate_one_or_more_chars_with_custom_character_mapping(char={char},code={code},code2={code2})",end="")
if char in unicode_to_ascii_custom_character_mapping: #if it's not found now, it's really not found
mapping = unicode_to_ascii_custom_character_mapping[char]
if DEBUG_UNIDECODECHAR_TRANSLATECHAR: primt(f"{Fore.GREEN} Found in mapping!",end="")
else:
if DEBUG_UNIDECODECHAR_TRANSLATECHAR: primt( f"{Fore.RED}Not found in mapping!",end="")
code2 = "code " + str(get_unicode_hex(char)).replace("\\","")
if code2 in unicode_to_ascii_custom_character_mapping:
if DEBUG_UNIDECODECHAR_TRANSLATECHAR: primt(f"{Fore.GREEN}{Style.BRIGHT}Found by 2nd-attempt code lookup!{Style.NORMAL}",end="")
mapping = unicode_to_ascii_custom_character_mapping[code2]
else:
if DEBUG_UNIDECODECHAR_TRANSLATECHAR: primt(f"{Style.BRIGHT}(Twice!)(code2={code2}){Style.NORMAL}",end="")
translated_chars.append(char)
continue
#mapping = unicode_to_ascii_custom_character_mapping[char]
if len(mapping) == 0: raise Exception("FATAL ERROR: ZERO MAPPING LENGTH")
if mode == "file" and len(mapping) > 1: mapping_number_to_use = 1
else: mapping_number_to_use = 0
translated_chars.append(mapping[mapping_number_to_use])
done = True # If any character is mapped, mark it as done
if DEBUG_UNIDECODECHAR_TRANSLATECHAR: primt("\n")
return ''.join(translated_chars), done
def is_emoji_character(char):
"""Checks if a character is an emoji."""
emoji_ranges = [
( '\u2600', '\u26FF'), # Miscellaneous Symbols
( '\u2700', '\u27BF'), # Dingbats
( '\uE000', '\uF8FF'), # Private Use Area
( '\uFE00', '\uFE0F'), # Variation Selectors
('\u1F000', '\u1F02B'), # Mahjong Tiles, Domino Tiles, Playing Cards
('\u1F030', '\u1F093'), # Enclosed Alphanumeric Supplement
('\u1F0A0', '\u1F0AE'), # Playing cards
('\u1F100', '\u1F1FF'), # Enclosed Alphanumeric Supplement
('\u1F200', '\u1F2FF'), # Enclosed Ideographic Supplement
('\u1F1E6', '\u1F1FF'), # Regional Indicator Symbols
('\u1F300', '\u1F5FF'), # Miscellaneous Symbols and Pictographs
('\u1F600', '\u1F64F'), # Emoticons
('\u1F680', '\u1F6FF'), # Transport and Map Symbols
('\u1F700', '\u1F77F'), # Alchemical Symbols
('\u1F780', '\u1F7FF'), # Geometric Shapes Extended
('\u1F800', '\u1F8FF'), # Supplemental Arrows-C
('\u1F900', '\u1F9FF'), # Supplemental Symbols and Pictographs
('\u1FA00', '\u1FA6F'), # Chess Symbols
('\u1FAB0', '\u1FAB6'), # Face in Cloud, Spiral, Hole, Rock, Wood, Hut
('\u1FAC0', '\u1FAC2'), # People Hugging, People with Bunny Ears, Person in Tuxedo
('\u1FAD0', '\u1FAD6'), # Heart on Fire, Mending Heart, Face Exhaling, Face with Spiral Eyes, Face in Clouds
('\u1FA70', '\u1FAFF'), # Symbols and Pictographs Extended-A
]
for start, end in emoji_ranges:
if start <= char <= end: return True
return False
is_emoji = is_emoji_character
def is_unicode_character(char):
"""Checks if a character is a valid Unicode character."""
unicode_range = ('\u0000', '\U0010FFFF')
return unicode_range[0] <= char <= unicode_range[1]
def translate_character_with_language_libraries(char,mode,filename="not given"): #pylint: disable=R0912,R0915
"""Translates a single character to its ASCII/roman equivalent.
Each character is processed individually, checking its unicode value.
The value is checked to see if it in the range of seveal specific languages.
For some languages, we use a language-specific proprietary library to convert back to ASCII/roman characters.
For some languages, we use a language-specific phoenetic mapping table to convert back to ASCII/roman characters.
Note that the final step, unidecode.unicode, is a multi-lingual catch-all.
For example, it is purported to remove accents over French/Spanish, vowels, & change Russian to phonetic equivalents
LANGUAGE SUPPORT:
Our list of addressed languages, even if only implicitly/partially addressed is, at the very least:
Arabic, Bengali, Chinese, English, French, Hindi, Japanese, Korean, Spanish, Russian, Thai
NEW LANGUAGES:
We do not need to actually and every language in existence.
We attempted to add the most common languages that have hard-to-romanize alphabets (usually non-"Western" languages)
Common languages that have easy-to-romanize alphabets are likely covered by unidecode.unicode.
"""
global DEBUG, DEBUG_CHAR, DEBUG_UNIDECODECHAR, DIE_ON_UNDECODEABLE_UNICODE_CHARACTER
char_for_primt = char.encode('utf-16', 'surrogatepass').decode('utf-16','ignore')
if DEBUG_CHAR:
try:
primt (f"- DEBUG: char is {Fore.YELLOW}{char}{Fore.WHITE}\tvalue {Fore.YELLOW}{get_unicode_hex(char)}{Fore.WHITE}{Style.NORMAL}",end="")
except Exception: #pylint: disable=W0718
primt (f"- DEBUG: char is {Fore.YELLOW}{char_for_primt}{Fore.WHITE}\tvalue {Fore.YELLOW}{get_unicode_hex(char)}{Fore.WHITE}{Style.NORMAL}",end="")
# First we check our custom mapping, our highest priority. It is hand-created and thought out.
char, done = translate_one_or_more_chars_with_custom_character_mapping(char,mode)
if DEBUG_CHAR: primt (f" \t... custom mapping: {Fore.YELLOW}{char}{Fore.WHITE}\tdone={done:1}",end="")
if done:
if DEBUG_CHAR: primt("")
return char
# if a character is still untranslated, then we check our various lingual libraries and phoenetic mapping tables:
is_emoji = False
caught = False
translate_return_value = ""
if '\u0600' <= char <= '\u06FF': caught,is_unicode=True,True; translate_return_value = translate_arabic___to_ascii(char) # if Arabic
elif '\u0900' <= char <= '\u097F': caught,is_unicode=True,True; translate_return_value = translate_hindi____to_ascii(char) # if Hindi
elif '\u0980' <= char <= '\u09FF': caught,is_unicode=True,True; translate_return_value = translate_bengali__to_ascii(char) # if Bengali
elif '\u0E01' <= char <= '\u0E5B': caught,is_unicode=True,True; translate_return_value = translate_thai_____to_ascii(char) # if Thai
elif '\u3040' <= char <= '\u30ff': caught,is_unicode=True,True; translate_return_value = translate_japanese_to_ascii(char) # if Japanese
elif '\u4e00' <= char <= '\u9fff': caught,is_unicode=True,True; translate_return_value = translate_chinese__to_ascii(char) # if Chinese
elif '\uac00' <= char <= '\ud7af': caught,is_unicode=True,True; translate_return_value = translate_korean___to_ascii(char) # if Korean
elif is_emoji_character(char) : # if Emoji
demojified = translate_emoji_to_ascii(char)
if demojified:
caught, is_emoji = True, True
translate_return_value = demojified
else:
caught, is_emoji = False, False
if DEBUG_UNIDECODECHAR: primt(f" | is_emomji?={is_emoji_character(char):1} | caight?={caught} | {char} {Style.BRIGHT}de-{Style.NORMAL}emojied is '{Fore.YELLOW}{demojified}'{Fore.WHITE} | translate_return_value={translate_return_value}",end="")
# if a character is even still untranlated, we need to use our catch-all code
# this library purports to fix things all kinds of things like: Spanish n-with-tilde will become an N,
# French c-with-a-hook will get hook removed, Russian is phonetically translated,
# but we fear it may return nothing if it doesn't have a great guess:
is_unicode = caught
if not caught:
is_unicode = is_unicode_character(char)
if not is_unicode:
translate_return_value = char
else:
if caught: char = translate_return_value
unidecodeChar = unidecode(char)
if DEBUG_UNIDECODECHAR:
if unidecodeChar == '': style_adjustment = f"{Fore.RED}"
else: style_adjustment = f"{Fore.WHITE}"
primt(f" | emoji?={is_emoji:1} | unicode?={is_unicode:1} | {char}\t{style_adjustment}uni{Style.BRIGHT}de{style_adjustment}{Style.NORMAL}coded is '{Fore.YELLOW}{unidecodeChar}{style_adjustment}'{Fore.WHITE}",end="")
if unidecodeChar == "":
translate_return_value = char
hex = get_unicode_hex(char)
unicodedata_decode = get_name_from_hex(hex)
if unicodedata_decode not in ["", char, translate_return_value]: #assign new character if it's actually a new character
translate_return_value = unicodedata_decode
else: #fairly unreachable code but comment out the if part and this can be a fun way to find un-manually-mapped characters to add more pleasant/customized mapping
message = f"{Fore.RED}{Style.BRIGHT}\n!!! FATAL DECODE ERROR: COULD NOT DECODE UNICODE CHARACTER OF {char} (unicode hex={hex}) !!!\nFilename = {filename}\nPlease add to custom mapping table at the bottom of fixUnicodeFilenames.py\nYou may need to copy and paste this character into google to find out what it actually is:\n%EDITOR% {sys.argv[0]}{Fore.WHITE}{Style.NORMAL}"
if DIE_ON_UNDECODEABLE_UNICODE_CHARACTER: raise Exception(message)
primt(message)
translate_return_value = "{" + translate_return_value + "}"
else:
translate_return_value = unidecodeChar
if DEBUG_CHAR or DEBUG_UNIDECODECHAR: primt("")
#If we are in file mode, we need to make one more pass because the previous code could have turned it into something bad due to a bug:
#First we check our custom mapping, our highest priority. It is hand-created and thought out.
#translate_return_value, _ = translate_one_or_more_chars_with_custom_character_mapping(translate_return_value,mode) #TODO evaluate whether it is safe to disable this now that we have internal 'tegrity checks for key values that would be invalid filenames
return translate_return_value
import emoji # emoji library
import romkan # Japanese library
from pypinyin import lazy_pinyin, Style as PypinyinStyle # Chinese library
from korean_romanizer.romanizer import Romanizer as KoreanRomanizer # Korean library
from pythainlp.transliterate import romanize as ThaiRomanize # Thai library
def translate_thai_____to_ascii(text): return ThaiRomanize(text) # Thai
def translate_japanese_to_ascii(char): return romkan.to_roma(char) # Japanese
def translate_chinese__to_ascii(char): return ''.join(lazy_pinyin(char, style=PypinyinStyle.TONE3)) # Chinese
def translate_bengali__to_ascii(text): return ''.join(bengali_to_english_phonetic.get(c, '_') for c in text) # Bengali (no library used)
def translate_arabic___to_ascii(text): return ''.join( arabic_to_english_phonetic.get(c, '_') for c in text) # Arabic (no library used)
def translate_hindi____to_ascii(text): return ''.join( hindi_to_english_phonetic.get(c, '_') for c in text) # Hindi (no library used)
def translate_korean___to_ascii(text): # Korean
try: retval = KoreanRomanizer(text).romanize()
except: retval = text
return retval
def translate_emoji_to_ascii(char):
demojized = emoji.demojize(char)
if demojized.startswith(':') and demojized.endswith(':'): return '{' + demojized[1:-1] + '}'
return demojized
def get_name_from_hex(unicode_hex):
primt(f"\n\nRunning get_name_from_hex({unicode_hex})")
unicode_hex_original = unicode_hex
unicode_hex = unicode_hex.replace('\\u', '').replace('\\U', '') # Remove the Unicode escape sequence part #added capital-U version for 2024/05/23 situation
primt(f"unicode_hex is now {unicode_hex}")
unicode_char = chr(int(unicode_hex, 16)) # Convert hex string to Unicode character
try:
return unicodedata.name(unicode_char)
except ValueError: # Raised when the character does not have a name
unicode_char = chr(int("000" + unicode_hex, 16)) # Convert hex string to Unicode character
try:
return unicodedata.name(unicode_char)
except ValueError: # Raised when the character does not have a name
unicode_char = chr(int("00" + unicode_hex, 16)) # Convert hex string to Unicode character
try:
return unicodedata.name(unicode_char)
except ValueError: # Raised when the character does not have a name
unicode_char = chr(int("0" + unicode_hex, 16)) # Convert hex string to Unicode character
try:
return unicodedata.name(unicode_char)
except ValueError: # Raised when the character does not have a name
#unicode_char = {{{TRY OTHER THINGS HERE}}} # Convert hex string to Unicode character
try:
return unicodedata.name(unicode_char)
except ValueError: # Raised when the character does not have a name
return f" [ERROR: get_name_from_hex ___ fail_for_hex={unicode_hex_original},char={unicode_char}] "
def ask_permission(old_name, new_name):
"""Asks the user for permission to rename a file."""
primt(f"\n{Fore.YELLOW}{Style.BRIGHT}***** Rename:" +
f"\n{Fore.RED }{Style.BRIGHT}From: {Style.NORMAL}{old_name}{Fore.CYAN}{Style.NORMAL}" +
f"\n{Fore.GREEN }{Style.BRIGHT} To: {Style.NORMAL}{new_name}{Fore.CYAN}{Style.NORMAL} " +
f"\n{Fore.YELLOW}{Style.BRIGHT}***** Rename?" +
f" { Fore.BLUE }{Style.BRIGHT}[{Fore.CYAN}Y{Fore.BLUE}/{Style.NORMAL}{Fore.CYAN}n{Style.BRIGHT}]{Style.NORMAL} ", end="")
clear_keyboard_buffer()
response = msvcrt.getch().decode().lower().strip()
primt(Style.BRIGHT, end="")
if response.lower() in ['y', 'yes', '']:
primt(f"{Fore.GREEN}Yes!", end="")
return True
primt(f"{Fore.RED}No!", end="")
return False
def clear_keyboard_buffer():
while msvcrt.kbhit(): msvcrt.getch()
#def rename_files_in_current_directory_last_ver_before_recursion(mode="file",automatic_mode=False,recursive_mode=False): #defaults to file mode
# """Renames all files in a directory, replacing unicode characters."""
# global DRY_RUN, DEBUG_ANNOUNCE_FILENAMES
# any_files_found_to_rename_at_all = False
# do_it_for_real = True
# automatic = False
# DRY_RUN = False
# permission = False
# directory = sys.argv[1] if len(sys.argv) > 1 else '.' #get all the files in the current dir...
# for filename in os.listdir(directory):
# filename_for_primt = filename.encode('utf-8','ignore')
# if DEBUG_ANNOUNCE_FILENAMES: primt(f"{Fore.CYAN}{Style.BRIGHT}* Processing file {filename}...{Style.NORMAL}{Fore.WHITE}")
# new_name = convert_to_ascii_filename_chracters(filename,mode) #this is where all the magic happens
#
# if filename != new_name:
# any_files_found_to_rename_at_all = True
# if automatic_mode:
# automatic = True
# do_it_for_real = True
# action_string = " Auto-Renamed"
# else:
# permission = ask_permission(filename, new_name)
# do_it_for_real = permission
# action_string = " Renamed" if permission is True else f"{Fore.RED}Did not rename"
# if DRY_RUN:
# do_it_for_real = False
#
# old_file = os.path.join(directory, filename)
# new_file = os.path.join(directory, new_name)
#
# new_new_file = last_minute_filename_cleanser(new_file) #if we've put invalid values in our mapping table without having run our tests, it can be possible to have to cleanse one more time. Also, some emoji libraries may decode into something invalid for filenames, and since we didn't test if all the decodings were valid, we must run it through a 2nd time for that possibility as well. It's unfortunate, but not expensive.
# if do_it_for_real:
# #os.rename(old_file, new_new_file) #would error if new folder already existed
# rename_folder_or_file_but_if_renamed_is_a_folder_that_already_exists_then_move_files_into_it_instead(old_file, new_new_file)
#
# primt("\n")
# if automatic: primt(f"\t{Fore.YELLOW} Automatic Run: {mode}")
# if DRY_RUN: primt(f"\t{Fore.YELLOW}" + "Dry Run: ")
# primt(f"{Fore.GREEN}{Style.NORMAL}\t{action_string}:" + f"\t{Fore.LIGHTBLACK_EX}{old_file} " +
# f"{Fore.CYAN}\n\t\t to:" + f"\t{Fore.GREEN}{new_new_file}{Style.NORMAL}\n\n\n")
# if not any_files_found_to_rename_at_all:
# primt(f"{Fore.RED}No files with unicode characters found.{Style.RESET_ALL}")
def rename_files_in_current_directory(mode="file",automatic_mode=False,recursive_mode=False):
"""Renames all files in a directory, replacing unicode characters."""
global DRY_RUN, DEBUG_ANNOUNCE_FILENAMES
any_files_found_to_rename_at_all = False
do_it_for_real = True
automatic = False
DRY_RUN = False
permission = False
directory = sys.argv[1] if len(sys.argv) > 1 else '.'
def process_directory(directory):
nonlocal any_files_found_to_rename_at_all, automatic
for filename in os.listdir(directory):
filename_for_primt = filename.encode('utf-8','ignore')
if DEBUG_ANNOUNCE_FILENAMES:
### without color-cycling:
#primt(f"{Fore.CYAN}{Style.BRIGHT}* Processing file {filename}...{Style.NORMAL}{Fore.WHITE}")
### with color-cycling:
original_print(f"* Processing file {filename}...")
for i in range(100): claire.tick(mode="fg") #TODO maybe consider the range(100) thing bad form haha but we're also testing another library
new_name = convert_to_ascii_filename_chracters(filename,mode)
if filename != new_name:
any_files_found_to_rename_at_all = True
if automatic_mode:
automatic = True
do_it_for_real = True
action_string = " Auto-Renamed"
else:
automatic = False
permission = ask_permission(filename, new_name)
do_it_for_real = permission
action_string = " Renamed" if permission is True else f"{Fore.RED}Did not rename"
if DRY_RUN:
do_it_for_real = False
old_file = os.path.join(directory, filename)
new_file = os.path.join(directory, new_name)
new_new_file = last_minute_filename_cleanser(new_file)
if do_it_for_real:
rename_folder_or_file_but_if_renamed_is_a_folder_that_already_exists_then_move_files_into_it_instead(old_file, new_new_file)
primt("\n")
if automatic: primt(f"\t{Fore.YELLOW} Automatic Run: {mode}")
if DRY_RUN: primt(f"\t{Fore.YELLOW}" + "Dry Run: ")
primt(f"{Fore.GREEN}{Style.NORMAL}\t{action_string}:" + f"\t{Fore.LIGHTBLACK_EX}{old_file} " +
f"{Fore.CYAN}\n\t\t to:" + f"\t{Fore.GREEN}{new_new_file}{Style.NORMAL}\n\n\n")
if recursive_mode:
for root, dirs, files in os.walk(directory):
process_directory(root)
else:
process_directory(directory)
if not any_files_found_to_rename_at_all:
primt(f"{Fore.RED}No files with unicode characters found.{Style.RESET_ALL}")
def rename_folder_or_file_but_if_renamed_is_a_folder_that_already_exists_then_move_files_into_it_instead(old_name, new_name):
if len(new_name) > 253: new_name = new_name.replace('{', '').replace('}', '')
if len(new_name) > 253: new_name = new_name.replace('(', '').replace(')', '')
if len(new_name) > 253: new_name = new_name.replace('[', '').replace(']', '')
if len(new_name) > 253: new_name = new_name.replace(' ', '')
if len(new_name) > 253: new_name = new_name[:253]
if not os.path.exists(new_name): # If the new folder doesn't exist, simply rename the old folder
os.rename(old_name, new_name)
else:
for filename in os.listdir(old_name): # If the new folder exists, move all files from the old folder to the new one
old_file_path = os.path.join(old_name, filename)
new_file_path = os.path.join(new_name, filename)
# If a file with the same name exists in the new directory, it will be replaced
# If you don't want this behavior, you can add a check here
shutil.move(old_file_path, new_file_path)
# Optionally, if you want to delete the old folder after moving all files
os.rmdir(old_name)
def last_minute_filename_cleanser_original(filename):
"""
This whole program could be just this one function, if we were not too picky.
"""
global INVALID_WINDOWS_FILENAME_CHARACTERS
if any(char in INVALID_WINDOWS_FILENAME_CHARACTERS for char in filename):
filename = convert_a_filename(filename,silent_if_unchanged=False) #TODO true
filename = filename.lstrip('.-') # Strip "." or "-" from the beginning of the filename
return filename
def last_minute_filename_cleanser(filename):
global INVALID_WINDOWS_FILENAME_CHARACTERS
leading_patterns = [".\\", "./", ".\\\\", ".//", "..\\", "..\\\\", "../", "..//"] # Define the leading patterns to exclude
for pattern in leading_patterns: # Check if the filename starts with any of the leading patterns
if filename.startswith(pattern):
stripped_filename = filename[len(pattern):] # Remove the leading pattern
break
else:
stripped_filename = filename # If no leading pattern found, use the original filename
if any(char in INVALID_WINDOWS_FILENAME_CHARACTERS for char in stripped_filename): # Perform the necessary processing on the stripped filename
stripped_filename = convert_a_filename(stripped_filename, silent_if_unchanged=True, silent_if_changed=True)
stripped_filename = stripped_filename.lstrip('.-') # Strip "." or "-" from the beginning of the filename
if stripped_filename != filename: # Restore the leading pattern, if it was stripped
stripped_filename = filename[:len(filename) - len(stripped_filename)] + stripped_filename
return stripped_filename
## Public calls:
def convert_a_string (string_to_convert ,silent_if_unchanged=False, silent_if_changed=False, silent=False): return just_convert_a_string( string_to_convert,"string",silent_if_unchanged=silent_if_unchanged,silent_if_changed=silent_if_changed,silent=silent)
def convert_a_filename(filename_to_convert,silent_if_unchanged=False, silent_if_changed=False, silent=False): return just_convert_a_string(filename_to_convert,"file" ,silent_if_unchanged=silent_if_unchanged,silent_if_changed=silent_if_changed,silent=silent)
def just_convert_a_string(string_to_convert,mode,silent_if_unchanged=False,silent_if_changed=False,silent=False):
global DIE_ON_UNDECODEABLE_UNICODE_CHARACTER
if __name__ != "__main__": DIE_ON_UNDECODEABLE_UNICODE_CHARACTER=False #only die when being run, not when being imported
# special handling for testing mode
if mode == "test":
run_internal_tests()
for temp_mode in ["file", "string"]:
primt (f"\n\n{Fore.YELLOW}{Style.BRIGHT}* Testing in mode {temp_mode}:{Style.NORMAL}\n")
primt ("Test result: " + just_convert_a_string(string_to_convert,temp_mode))
return ":)"
# special handling for script mdoe
if mode == "script":
create_script_to_define_emoji_characters()
sys.exit(0)
return ":)"
# actually convert the string
romanized_string = convert_to_ascii_filename_chracters(string_to_convert,mode) #...which we then fix the same way we would fix our filenames
# print out the ch ange if we are instructed to do so
if silent or (silent_if_unchanged and string_to_convert == romanized_string) or (silent_if_changed and string_to_convert != romanized_string):
pass #don't primt
else:
primt(f"{Fore.RED}Old string: {string_to_convert}")
primt(f"{Fore.GREEN}New string: {romanized_string }")
return romanized_string
#from emoji.unicode_codes import EMOJI_DATA
#
#def create_script_to_define_emoji_characters():
# for emoji, emoji_data in EMOJI_DATA.items():
# # Fetch the emoji name
# emoji_name = emoji_data.get('en', '')
# emoji_name = emoji_name.upper().replace(' ', '_').replace(':', '').replace('-', '_')
#
# # Get the unicode code points and convert them to decimal
# emoji_codes = emoji.encode('unicode_escape').decode('ASCII').split('\\')[1:]
#
# # Create the output string
# output_string = f"SET EMOJI_{emoji_name}="
# for code in reversed(emoji_codes): # reversed added here
# if code.startswith('0'):
# value = int(code, 16)
# output_string += f"%@CHAR[{value}]"
# elif code.startswith('U'):
# value = int(code[1:], 16)
# output_string += f"%@CHAR[{value}]"
# elif code == 'ufe0f':
# # This is a variation selector, handle it accordingly
# output_string += "+%@CHAR[65039]" # 65039 is decimal equivalent of 'U+FE0F'
# elif code == 'u200d':
# # This is a Zero Width Joiner, handle it accordingly
# output_string += "+%@CHAR[8205]" # 8205 is decimal equivalent of 'U+200D'
# else:
# # Unknown code, handle it as you see fit
# primt(f"Unknown code encountered: {code}")
#
# primt(output_string)
def create_script_to_define_emoji_characters_1():
primt("EMOJI_ENVIRONMENT_VARIABLES_CREATED_BY=fix_unicode_files.py script")
import ctypes
from emoji.unicode_codes import EMOJI_DATA
processed_emojis = set() # Set to track processed emojis
output_strings = [] # List to store the output strings
processed_output_strings = set() # Set to track processed output strings
for emoji, emoji_data in EMOJI_DATA.items():
# Fetch the base emoji without any skin tone variation
base_emoji = emoji.split('\u200d')[0]
# Check if the base emoji has already been processed
if base_emoji not in processed_emojis:
processed_emojis.add(base_emoji)
emoji_name_meat = emoji_data['en'].upper().replace(' ', '_').replace(':', '').replace('-', '_').replace("'", '').replace('SKIN_TONE', 'SKIN').replace('&', '_AND_')
# Check if the current emoji is fully qualified
if emoji_data['status'] == 'fully_qualified':
emoji_name = f"EMOJI_{emoji_name_meat}"
else:
emoji_name = f"EMOJI_{emoji_name_meat}_UNQUALIFIED"
# Convert the emoji into a ctypes wide string
# Then cast it to a pointer to short (16-bit) integers, and fetch the values
emoji_code_units = ctypes.cast(ctypes.c_wchar_p(emoji), ctypes.POINTER(ctypes.c_uint16))
# Create the output string
output_string = f"{emoji_name}="
for i in range(2): # two UTF-16 code units
output_string += f"%@CHAR[{emoji_code_units[i]}]"
output_strings.append(output_string)
# Check and print the output strings
for output_string in output_strings:
if output_string not in processed_output_strings:
processed_output_strings.add(output_string)
# Print the output strings
for output_string in processed_output_strings:
primt(output_string)
def create_script_to_define_emoji_charactersDECENTBUTPROBLEMATICAF():
primt("EMOJI_ENVIRONMENT_VARIABLES_CREATED_BY=fix_unicode_files.py script")
import ctypes
from emoji.unicode_codes import EMOJI_DATA
processed_emojis = set() # Set to track processed emojis
qualified_emojis = set() # Set to track qualified emojis
output_strings = [] # List to store the output strings
for emoji, emoji_data in EMOJI_DATA.items():
# Fetch the base emoji without any skin tone variation
base_emoji = emoji.split('\u200d')[0]
# Check if the base emoji has already been processed
if base_emoji not in processed_emojis:
processed_emojis.add(base_emoji)
emoji_name_meat = emoji_data['en'].upper().replace(' ', '_').replace(':', '').replace('-', '_').replace("'", '').replace('SKIN_TONE', 'SKIN').replace('&', '_AND_')
# Check if the current emoji is fully qualified
if emoji_data['status'] == 'fully_qualified':
qualified_emojis.add(emoji_name_meat)
emoji_name = f"EMOJI_{emoji_name_meat}"
qualified_output_string = None
else:
emoji_name = f"EMOJI_{emoji_name_meat}_UNQUALIFIED"
qualified_output_string = f"EMOJI_{emoji_name_meat}"
# Convert the emoji into a ctypes wide string
# Then cast it to a pointer to short (16-bit) integers, and fetch the values
emoji_code_units = ctypes.cast(ctypes.c_wchar_p(emoji), ctypes.POINTER(ctypes.c_uint16))
# Create the output string
output_string = f"{emoji_name}="
for i in range(2): # two UTF-16 code units
output_string += f"%@CHAR[{emoji_code_units[i]}]"
output_strings.append(output_string)
# Append the qualified output string if available
if qualified_output_string:
qualified_output_string += f"=%@CHAR[{emoji_code_units[0]}]%@CHAR[{emoji_code_units[1]}]"
output_strings.append(qualified_output_string)
# Print the output strings
for output_string in output_strings:
primt(output_string)
def create_script_to_define_emoji_characters_tried_without_gpt_got_5718():
primt("EMOJI_ENVIRONMENT_VARIABLES_CREATED_BY=fix_unicode_files.py script")
import ctypes
from emoji.unicode_codes import EMOJI_DATA
processed_emojis = set() # Set to track processed emojis
qualified_emojis = set() # Set to track qualified emojis
unqualified_emojis = set() # Set to track qualified emojis
rights = set() # set to track right half of = in output file so we don't make duplicates
output_strings = [] # List to store the output strings
for emoji, emoji_data in EMOJI_DATA.items():
# Fetch the base emoji without any skin tone variation
base_emoji = emoji.split('\u200d')[0]
# Check if the base emoji has already been processed
if base_emoji not in processed_emojis:
processed_emojis.add(base_emoji)
emoji_name_meat = emoji_data['en'].upper().replace(' ', '_').replace(':', '').replace('-', '_').replace("'", '').replace('SKIN_TONE', 'SKIN').replace('&', '_AND_')
# Check if the current emoji is fully qualified
if emoji_data['status'] == 'fully_qualified':
if emoji_name_meat in qualified_emojis:
continue
qualified_emojis.add(emoji_name_meat)
emoji_name = f"EMOJI_{emoji_name_meat}"
qualified_output_string = None
else:
if emoji_name_meat in unqualified_emojis:
continue
unqualified_emojis.add(emoji_name_meat)
emoji_name = f"EMOJI_{emoji_name_meat}_UNQUALIFIED"
qualified_output_string = f"EMOJI_{emoji_name_meat}"
# Convert the emoji into a ctypes wide string
# Then cast it to a pointer to short (16-bit) integers, and fetch the values
emoji_code_units = ctypes.cast(ctypes.c_wchar_p(emoji), ctypes.POINTER(ctypes.c_uint16))
# Create the output string
right = ""
for i in range(2): # two UTF-16 code units
right += f"%@CHAR[{emoji_code_units[i]}]"
output_string = f"{emoji_name}={right}"
if right in rights:
continue
rights.add(right)
if output_string in output_strings:
continue
output_strings.append(output_string)
# Append the qualified output string if available
if qualified_output_string:
qualified_output_string += f"=%@CHAR[{emoji_code_units[0]}]%@CHAR[{emoji_code_units[1]}]"
output_strings.append(qualified_output_string)
# Print the output strings
printed = set()
for output_string in output_strings:
if output_string in printed: continue
printed.add(output_string)
primt(output_string)
def create_script_to_define_emoji_characters_got_3106_much_better():
primt("EMOJI_ENVIRONMENT_VARIABLES_CREATED_BY=fix_unicode_files.py script")
import ctypes
from emoji.unicode_codes import EMOJI_DATA
processed_emojis = set() # Set to track processed emojis
rights = set() # Set to track right half of = in output file so we don't make duplicates
output_strings = set() # Set to handle duplicate output strings
for emoji, emoji_data in EMOJI_DATA.items():
# Fetch the base emoji without any skin tone variation
base_emoji = emoji.split('\u200d')[0]
# Check if the base emoji has already been processed
if base_emoji not in processed_emojis:
processed_emojis.add(base_emoji)
emoji_name_meat = emoji_data['en'].upper().replace(' ', '_').replace(':', '').replace('-', '_').replace("'", '').replace('SKIN_TONE', 'SKIN').replace('&', '_AND_')
# Convert the emoji into a ctypes wide string
# Then cast it to a pointer to short (16-bit) integers, and fetch the values
emoji_code_units = ctypes.cast(ctypes.c_wchar_p(emoji), ctypes.POINTER(ctypes.c_uint16))
# Create the output string
right = ""
for i in range(2): # two UTF-16 code units
right += f"%@CHAR[{emoji_code_units[i]}]"
if right in rights:
continue
rights.add(right)
# Check if the current emoji is fully qualified
if emoji_data['status'] == 'fully_qualified':
emoji_name = f"EMOJI_{emoji_name_meat}"
else:
emoji_name = f"EMOJI_{emoji_name_meat}_UNQUALIFIED"
output_string = f"{emoji_name}={right}"
output_strings.add(output_string) # Add to a set to handle duplicates
# Print the output strings
for output_string in output_strings:
primt(output_string)
# thread about this: https://jpsoft.com/forums/threads/1431-emoji-environment-variables-for-your-echoing-convenience.11618/
def create_script_to_define_emoji_characters(): #2860, 1431 unique
primt("EMOJI_ENVIRONMENT_VARIABLES_CREATED_BY=fix_unicode_files.py script")
import ctypes
from emoji.unicode_codes import EMOJI_DATA
processed_emojis = set() # Set to track processed emojis
rights = set() # Set to track right half of = in output file so we don't make duplicates
output_strings = set() # Set to handle duplicate output strings
for emoji, emoji_data in EMOJI_DATA.items():
# Fetch the base emoji without any skin tone variation
base_emoji = emoji.split('\u200d')[0]
emoji_name_meat = emoji_data['en'].upper().replace(' ', '_').replace(':', '').replace('-', '_').replace("'", '').replace('SKIN_TONE', 'SKIN').replace('&', '_AND_').replace('�','')