-
-
Notifications
You must be signed in to change notification settings - Fork 859
/
config.d
2261 lines (2039 loc) · 96.1 KB
/
config.d
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
// What is this module called?
module config;
// What does this module require to function?
import core.stdc.stdlib: EXIT_SUCCESS, EXIT_FAILURE, exit;
import std.stdio;
import std.process;
import std.regex;
import std.string;
import std.algorithm.searching;
import std.algorithm.sorting: sort;
import std.file;
import std.conv;
import std.path;
import std.getopt;
import std.format;
import std.ascii;
import std.datetime;
// What other modules that we have created do we need to import?
import log;
import util;
class ApplicationConfig {
// Application default values - these do not change
// - Compile time regex
immutable auto configRegex = ctRegex!(`^(\w+)\s*=\s*"(.*)"\s*$`);
// - Default directory to store data
immutable string defaultSyncDir = "~/OneDrive";
// - Default Directory Permissions
immutable long defaultDirectoryPermissionMode = 700;
// - Default File Permissions
immutable long defaultFilePermissionMode = 600;
// - Default types of files to skip
// v2.0.x - 2.4.x: ~*|.~*|*.tmp
// v2.5.x : ~*|.~*|*.tmp|*.swp|*.partial
immutable string defaultSkipFile = "~*|.~*|*.tmp|*.swp|*.partial";
// - Default directories to skip (default is skip none)
immutable string defaultSkipDir = "";
// - Default log directory
immutable string defaultLogFileDir = "/var/log/onedrive";
// - Default configuration directory
immutable string defaultConfigDirName = "~/.config/onedrive";
// Microsoft Requirements
// - Default Application ID (abraunegg)
immutable string defaultApplicationId = "d50ca740-c83f-4d1b-b616-12c519384f0c";
// - Microsoft User Agent ISV Tag
immutable string isvTag = "ISV";
// - Microsoft User Agent Company name
immutable string companyName = "abraunegg";
// - Microsoft Application name as per Microsoft Azure application registration
immutable string appTitle = "OneDrive Client for Linux";
// Comply with OneDrive traffic decoration requirements
// https://docs.microsoft.com/en-us/sharepoint/dev/general-development/how-to-avoid-getting-throttled-or-blocked-in-sharepoint-online
// - Identify as ISV and include Company Name, App Name separated by a pipe character and then adding Version number separated with a slash character
//immutable string defaultUserAgent = isvTag ~ "|" ~ companyName ~ "|" ~ appTitle ~ "/" ~ strip(import("version"));
immutable string defaultUserAgent = isvTag ~ "|" ~ companyName ~ "|" ~ appTitle ~ "/" ~ "v2.5.0-alpha-3";
// HTTP Struct items, used for configuring HTTP()
// Curl Timeout Handling
// libcurl dns_cache_timeout timeout
immutable int defaultDnsTimeout = 60;
// Connect timeout for HTTP|HTTPS connections
immutable int defaultConnectTimeout = 30;
// Default data timeout for HTTP
// curl.d has a default of: _defaultDataTimeout = dur!"minutes"(2);
immutable int defaultDataTimeout = 240;
// Maximum time any operation is allowed to take
// This includes dns resolution, connecting, data transfer, etc.
immutable int defaultOperationTimeout = 3600;
// Specify what IP protocol version should be used when communicating with OneDrive
immutable int defaultIpProtocol = 0; // 0 = IPv4 + IPv6, 1 = IPv4 Only, 2 = IPv6 Only
// Specify how many redirects should be allowed
immutable int defaultMaxRedirects = 5;
// Azure Active Directory & Graph Explorer Endpoints
// - Global & Default
immutable string globalAuthEndpoint = "https://login.microsoftonline.com";
immutable string globalGraphEndpoint = "https://graph.microsoft.com";
// - US Government L4
immutable string usl4AuthEndpoint = "https://login.microsoftonline.us";
immutable string usl4GraphEndpoint = "https://graph.microsoft.us";
// - US Government L5
immutable string usl5AuthEndpoint = "https://login.microsoftonline.us";
immutable string usl5GraphEndpoint = "https://dod-graph.microsoft.us";
// - Germany
immutable string deAuthEndpoint = "https://login.microsoftonline.de";
immutable string deGraphEndpoint = "https://graph.microsoft.de";
// - China
immutable string cnAuthEndpoint = "https://login.chinacloudapi.cn";
immutable string cnGraphEndpoint = "https://microsoftgraph.chinacloudapi.cn";
// Application items that depend on application run-time environment, thus cannot be immutable
// Public variables
// Was the application just authorised - paste of response uri
bool applicationAuthorizeResponseUri = false;
// Store the 'refresh_token' file path
string refreshTokenFilePath = "";
// Store the refreshToken for use within the application
string refreshToken;
// Store the accessTokenExpiration for use within the application
SysTime accessTokenExpiration;
// Store the current accessToken for use within the application
string accessToken;
// Store the 'session_upload.CRC32-HASH' file path
string uploadSessionFilePath = "";
bool apiWasInitialised = false;
bool syncEngineWasInitialised = false;
string accountType;
string defaultDriveId;
string defaultRootId;
ulong remainingFreeSpace = 0;
bool quotaAvailable = true;
bool quotaRestricted = false;
bool fullScanTrueUpRequired = false;
bool surpressLoggingOutput = false;
// This is the value that needs testing when we are actually downloading and uploading data
ulong concurrentThreads = 16;
// All application run-time paths are formulated from this as a set of defaults
// - What is the home path of the actual 'user' that is running the application
string defaultHomePath = "";
// - What is the config path for the application. By default, this is ~/.config/onedrive but can be overridden by using --confdir
private string configDirName = defaultConfigDirName;
// - In case we have to use a system config directory such as '/etc/onedrive' or similar, store that path in this variable
private string systemConfigDirName = "";
// - Store the configured converted octal value for directory permissions
private int configuredDirectoryPermissionMode;
// - Store the configured converted octal value for file permissions
private int configuredFilePermissionMode;
// - Store the 'delta_link' file path
private string deltaLinkFilePath = "";
// - Store the 'items.sqlite3' file path
string databaseFilePath = "";
// - Store the 'items-dryrun.sqlite3' file path
string databaseFilePathDryRun = "";
// - Store the user 'config' file path
private string userConfigFilePath = "";
// - Store the system 'config' file path
private string systemConfigFilePath = "";
// - What is the 'config' file path that will be used?
private string applicableConfigFilePath = "";
// - Store the 'sync_list' file path
string syncListFilePath = "";
// - Store the 'business_shared_items' file path
string businessSharedItemsFilePath = "";
// Hash files so that we can detect when the configuration has changed, in items that will require a --resync
private string configHashFile = "";
private string configBackupFile = "";
private string syncListHashFile = "";
private string businessSharedItemsHashFile = "";
// Store the actual 'runtime' hash
private string currentConfigHash = "";
private string currentSyncListHash = "";
private string currentBusinessSharedItemsHash = "";
// Store the previous config files hash values (file contents)
private string previousConfigHash = "";
private string previousSyncListHash = "";
private string previousBusinessSharedItemsHash = "";
// Store items that come in from the 'config' file, otherwise these need to be set the the defaults
private string configFileSyncDir = defaultSyncDir;
private string configFileSkipFile = defaultSkipFile;
private string configFileSkipDir = ""; // Default here is no directories are skipped
private string configFileDriveId = ""; // Default here is that no drive id is specified
private bool configFileSkipDotfiles = false;
private bool configFileSkipSymbolicLinks = false;
private bool configFileSyncBusinessSharedItems = false;
// File permission values (set via initialise function)
private int convertedPermissionValue;
// Array of values that are the actual application runtime configuration
// The values stored in these array's are the actual application configuration which can then be accessed by getValue & setValue
string[string] stringValues;
long[string] longValues;
bool[string] boolValues;
bool shellEnvironmentSet = false;
// Initialise the application configuration
bool initialise(string confdirOption) {
// Default runtime configuration - entries in config file ~/.config/onedrive/config or derived from variables above
// An entry here means it can be set via the config file if there is a coresponding entry, read from config and set via update_from_args()
// The below becomes the 'default' application configuration before config file and/or cli options are overlayed on top
// - Set the required default values
stringValues["application_id"] = defaultApplicationId;
stringValues["log_dir"] = defaultLogFileDir;
stringValues["skip_dir"] = defaultSkipDir;
stringValues["skip_file"] = defaultSkipFile;
stringValues["sync_dir"] = defaultSyncDir;
stringValues["user_agent"] = defaultUserAgent;
// - The 'drive_id' is used when we specify a specific OneDrive ID when attempting to sync Shared Folders and SharePoint items
stringValues["drive_id"] = "";
// Support National Azure AD endpoints as per https://docs.microsoft.com/en-us/graph/deployments
// By default, if empty, use standard Azure AD URL's
// Will support the following options:
// - USL4
// AD Endpoint: https://login.microsoftonline.us
// Graph Endpoint: https://graph.microsoft.us
// - USL5
// AD Endpoint: https://login.microsoftonline.us
// Graph Endpoint: https://dod-graph.microsoft.us
// - DE
// AD Endpoint: https://portal.microsoftazure.de
// Graph Endpoint: https://graph.microsoft.de
// - CN
// AD Endpoint: https://login.chinacloudapi.cn
// Graph Endpoint: https://microsoftgraph.chinacloudapi.cn
stringValues["azure_ad_endpoint"] = "";
// Support single-tenant applications that are not able to use the "common" multiplexer
stringValues["azure_tenant_id"] = "";
// - Store how many times was --verbose added
longValues["verbose"] = log.verbose; // might also be initialised by the first getopt call!
// - The amount of time (seconds) between monitor sync loops
longValues["monitor_interval"] = 300;
// - What size of file should be skipped?
longValues["skip_size"] = 0;
// - How many 'loops' when using --monitor, before we print out high frequency recurring items?
longValues["monitor_log_frequency"] = 12;
// - Number of N sync runs before performing a full local scan of sync_dir
// By default 12 which means every ~60 minutes a full disk scan of sync_dir will occur
// 'monitor_interval' * 'monitor_fullscan_frequency' = 3600 = 1 hour
longValues["monitor_fullscan_frequency"] = 12;
// - Number of children in a path that is locally removed which will be classified as a 'big data delete'
longValues["classify_as_big_delete"] = 1000;
// - Configure the default folder permission attributes for newly created folders
longValues["sync_dir_permissions"] = defaultDirectoryPermissionMode;
// - Configure the default file permission attributes for newly created file
longValues["sync_file_permissions"] = defaultFilePermissionMode;
// - Configure download / upload rate limits
longValues["rate_limit"] = 0;
// - To ensure we do not fill up the load disk, how much disk space should be reserved by default
longValues["space_reservation"] = 50 * 2^^20; // 50 MB as Bytes
// HTTPS & CURL Operation Settings
// - Maximum time an operation is allowed to take
// This includes dns resolution, connecting, data transfer, etc.
longValues["operation_timeout"] = defaultOperationTimeout;
// libcurl dns_cache_timeout timeout
longValues["dns_timeout"] = defaultDnsTimeout;
// Timeout for HTTPS connections
longValues["connect_timeout"] = defaultConnectTimeout;
// Timeout for activity on a HTTPS connection
longValues["data_timeout"] = defaultDataTimeout;
// What IP protocol version should be used when communicating with OneDrive
longValues["ip_protocol_version"] = defaultIpProtocol; // 0 = IPv4 + IPv6, 1 = IPv4 Only, 2 = IPv6 Only
// - Do we wish to upload only?
boolValues["upload_only"] = false;
// - Do we need to check for the .nomount file on the mount point?
boolValues["check_nomount"] = false;
// - Do we need to check for the .nosync file anywhere?
boolValues["check_nosync"] = false;
// - Do we wish to download only?
boolValues["download_only"] = false;
// - Do we disable notifications?
boolValues["disable_notifications"] = false;
// - Do we bypass all the download validation?
// This is critically important not to disable, but because of SharePoint 'feature' can be highly desirable to enable
boolValues["disable_download_validation"] = false;
// - Do we bypass all the upload validation?
// This is critically important not to disable, but because of SharePoint 'feature' can be highly desirable to enable
boolValues["disable_upload_validation"] = false;
// - Do we enable logging?
boolValues["enable_logging"] = false;
// - Do we force HTTP 1.1 for connections to the OneDrive API
// By default we use the curl library default, which should be HTTP2 for most operations governed by the OneDrive API
boolValues["force_http_11"] = false;
// - Do we treat the local file system as the source of truth for our data?
boolValues["local_first"] = false;
// - Do we ignore local file deletes, so that all files are retained online?
boolValues["no_remote_delete"] = false;
// - Do we skip symbolic links?
boolValues["skip_symlinks"] = false;
// - Do we enable debugging for all HTTPS flows. Critically important for debugging API issues.
boolValues["debug_https"] = false;
// - Do we skip .files and .folders?
boolValues["skip_dotfiles"] = false;
// - Do we perform a 'dry-run' with no local or remote changes actually being performed?
boolValues["dry_run"] = false;
// - Do we sync all the files in the 'sync_dir' root?
boolValues["sync_root_files"] = false;
// - Do we delete source after successful transfer?
boolValues["remove_source_files"] = false;
// - Do we perform strict matching for skip_dir?
boolValues["skip_dir_strict_match"] = false;
// - Do we perform a --resync?
boolValues["resync"] = false;
// - resync now needs to be acknowledged based on the 'risk' of using it
boolValues["resync_auth"] = false;
// - Ignore data safety checks and overwrite local data rather than preserve & rename
// This is a config file option ONLY
boolValues["bypass_data_preservation"] = false;
// - Allow enable / disable of the syncing of OneDrive Business Shared items (files & folders) via configuration file
boolValues["sync_business_shared_items"] = false;
// - Log to application output running configuration values
boolValues["display_running_config"] = false;
// - Configure read-only authentication scope
boolValues["read_only_auth_scope"] = false;
// - Flag to cleanup local files when using --download-only
boolValues["cleanup_local_files"] = false;
// Webhook Feature Options
boolValues["webhook_enabled"] = false;
stringValues["webhook_public_url"] = "";
stringValues["webhook_listening_host"] = "";
longValues["webhook_listening_port"] = 8888;
longValues["webhook_expiration_interval"] = 600;
longValues["webhook_renewal_interval"] = 300;
longValues["webhook_retry_interval"] = 60;
// Print in debug the application version as soon as possible
//log.vdebug("Application Version: ", strip(import("version")));
string tempVersion = "v2.5.0-alpha-3" ~ " GitHub version: " ~ strip(import("version"));
log.vdebug("Application Version: ", tempVersion);
// EXPAND USERS HOME DIRECTORY
// Determine the users home directory.
// Need to avoid using ~ here as expandTilde() below does not interpret correctly when running under init.d or systemd scripts
// Check for HOME environment variable
if (environment.get("HOME") != ""){
// Use HOME environment variable
log.vdebug("runtime_environment: HOME environment variable detected, expansion of '~' should be possible");
defaultHomePath = environment.get("HOME");
shellEnvironmentSet = true;
} else {
if ((environment.get("SHELL") == "") && (environment.get("USER") == "")){
// No shell is set or username - observed case when running as systemd service under CentOS 7.x
log.vdebug("runtime_environment: No HOME, SHELL or USER environment variable configuration detected. Expansion of '~' not possible");
defaultHomePath = "/root";
shellEnvironmentSet = false;
} else {
// A shell & valid user is set, but no HOME is set, use ~ which can be expanded
log.vdebug("runtime_environment: SHELL and USER environment variable detected, expansion of '~' should be possible");
defaultHomePath = "~";
shellEnvironmentSet = true;
}
}
// outcome of setting defaultHomePath
log.vdebug("runtime_environment: Calculated defaultHomePath: ", defaultHomePath);
// DEVELOPER OPTIONS
// display_memory = true | false
// - It may be desirable to display the memory usage of the application to assist with diagnosing memory issues with the application
// - This is especially beneficial when debugging or performing memory tests with Valgrind
boolValues["display_memory"] = false;
// monitor_max_loop = long value
// - It may be desirable to, when running in monitor mode, force monitor mode to 'quit' after X number of loops
// - This is especially beneficial when debugging or performing memory tests with Valgrind
longValues["monitor_max_loop"] = 0;
// display_sync_options = true | false
// - It may be desirable to see what options are being passed in to performSync() without enabling the full verbose debug logging
boolValues["display_sync_options"] = false;
// force_children_scan = true | false
// - Force client to use /children rather than /delta to query changes on OneDrive
// - This option flags nationalCloudDeployment as true, forcing the client to act like it is using a National Cloud Deployment model
boolValues["force_children_scan"] = false;
// display_processing_time = true | false
// - Enabling this option will add function processing times to the console output
// - This then enables tracking of where the application is spending most amount of time when processing data when users have questions re performance
boolValues["display_processing_time"] = false;
// Function variables
string configDirBase;
string systemConfigDirBase;
bool configurationInitialised = false;
// Initialise the application configuration, using the provided --confdir option was passed in
if (!confdirOption.empty) {
// A CLI 'confdir' was passed in
// Clean up any stray " .. these should not be there for correct process handling of the configuration option
confdirOption = strip(confdirOption,"\"");
log.vdebug("configDirName: CLI override to set configDirName to: ", confdirOption);
if (canFind(confdirOption,"~")) {
// A ~ was found
log.vdebug("configDirName: A '~' was found in configDirName, using the calculated 'defaultHomePath' to replace '~'");
configDirName = defaultHomePath ~ strip(confdirOption,"~","~");
} else {
configDirName = confdirOption;
}
} else {
// Determine the base directory relative to which user specific configuration files should be stored
if (environment.get("XDG_CONFIG_HOME") != ""){
log.vdebug("configDirBase: XDG_CONFIG_HOME environment variable set");
configDirBase = environment.get("XDG_CONFIG_HOME");
} else {
// XDG_CONFIG_HOME does not exist on systems where X11 is not present - ie - headless systems / servers
log.vdebug("configDirBase: WARNING - no XDG_CONFIG_HOME environment variable set");
configDirBase = buildNormalizedPath(buildPath(defaultHomePath, ".config"));
// Also set up a path to pre-shipped shared configs (which can be overridden by supplying a config file in userspace)
systemConfigDirBase = "/etc";
}
// Output configDirBase calculation
log.vdebug("configDirBase: ", configDirBase);
// Set the calculated application configuration directory
log.vdebug("configDirName: Configuring application to use calculated config path");
// configDirBase contains the correct path so we do not need to check for presence of '~'
configDirName = buildNormalizedPath(buildPath(configDirBase, "onedrive"));
// systemConfigDirBase contains the correct path so we do not need to check for presence of '~'
systemConfigDirName = buildNormalizedPath(buildPath(systemConfigDirBase, "onedrive"));
}
// Configuration directory should now have been correctly identified
if (!exists(configDirName)) {
// create the directory
mkdirRecurse(configDirName);
// Configure the applicable permissions for the folder
configDirName.setAttributes(returnRequiredDirectoryPermisions());
} else {
// The config path exists
// The path that exists must be a directory, not a file
if (!isDir(configDirName)) {
if (!confdirOption.empty) {
// the configuration path was passed in by the user .. user error
log.error("ERROR: --confdir entered value is an existing file instead of an existing directory");
} else {
// other error
log.error("ERROR: " ~ confdirOption ~ " is a file rather than a directory");
}
// Must exit
exit(EXIT_FAILURE);
}
}
// Update application set variables based on configDirName
// - What is the full path for the 'refresh_token'
refreshTokenFilePath = buildNormalizedPath(buildPath(configDirName, "refresh_token"));
// - What is the full path for the 'delta_link'
deltaLinkFilePath = buildNormalizedPath(buildPath(configDirName, "delta_link"));
// - What is the full path for the 'items.sqlite3' - the database cache file
databaseFilePath = buildNormalizedPath(buildPath(configDirName, "items.sqlite3"));
// - What is the full path for the 'items-dryrun.sqlite3' - the dry-run database cache file
databaseFilePathDryRun = buildNormalizedPath(buildPath(configDirName, "items-dryrun.sqlite3"));
// - What is the full path for the 'resume_upload'
uploadSessionFilePath = buildNormalizedPath(buildPath(configDirName, "session_upload"));
// - What is the full path for the 'sync_list' file
syncListFilePath = buildNormalizedPath(buildPath(configDirName, "sync_list"));
// - What is the full path for the 'config' - the user file to configure the application
userConfigFilePath = buildNormalizedPath(buildPath(configDirName, "config"));
// - What is the full path for the system 'config' file if it is required
systemConfigFilePath = buildNormalizedPath(buildPath(systemConfigDirName, "config"));
// - What is the full path for the 'business_shared_items'
businessSharedItemsFilePath = buildNormalizedPath(buildPath(configDirName, "business_shared_items"));
// To determine if any configuration items has changed, where a --resync would be required, we need to have a hash file for the following items
// - 'config.backup' file
// - applicable 'config' file
// - 'sync_list' file
// - 'business_shared_items' file
configBackupFile = buildNormalizedPath(buildPath(configDirName, ".config.backup"));
configHashFile = buildNormalizedPath(buildPath(configDirName, ".config.hash"));
syncListHashFile = buildNormalizedPath(buildPath(configDirName, ".sync_list.hash"));
businessSharedItemsHashFile = buildNormalizedPath(buildPath(configDirName, ".business_shared_items.hash"));
// Debug Output for application set variables based on configDirName
log.vdebug("refreshTokenFilePath = ", refreshTokenFilePath);
log.vdebug("deltaLinkFilePath = ", deltaLinkFilePath);
log.vdebug("databaseFilePath = ", databaseFilePath);
log.vdebug("databaseFilePathDryRun = ", databaseFilePathDryRun);
log.vdebug("uploadSessionFilePath = ", uploadSessionFilePath);
log.vdebug("userConfigFilePath = ", userConfigFilePath);
log.vdebug("syncListFilePath = ", syncListFilePath);
log.vdebug("systemConfigFilePath = ", systemConfigFilePath);
log.vdebug("configBackupFile = ", configBackupFile);
log.vdebug("configHashFile = ", configHashFile);
log.vdebug("syncListHashFile = ", syncListHashFile);
log.vdebug("businessSharedItemsFilePath = ", businessSharedItemsFilePath);
log.vdebug("businessSharedItemsHashFile = ", businessSharedItemsHashFile);
// Configure the Hash and Backup File Permission Value
string valueToConvert = to!string(defaultFilePermissionMode);
auto convertedValue = parse!long(valueToConvert, 8);
convertedPermissionValue = to!int(convertedValue);
// Initialise the application using the configuration file if it exists
if (!exists(userConfigFilePath)) {
// 'user' configuration file does not exist
// Is there a system configuration file?
if (!exists(systemConfigFilePath)) {
// 'system' configuration file does not exist
log.vlog("No user or system config file found, using application defaults");
applicableConfigFilePath = userConfigFilePath;
configurationInitialised = true;
} else {
// 'system' configuration file exists
// can we load the configuration file without error?
if (loadConfigFile(systemConfigFilePath)) {
// configuration file loaded without error
log.log("System configuration file successfully loaded");
// Set 'applicableConfigFilePath' to equal the 'config' we loaded
applicableConfigFilePath = systemConfigFilePath;
// Update the configHashFile path value to ensure we are using the system 'config' file for the hash
configHashFile = buildNormalizedPath(buildPath(systemConfigDirName, ".config.hash"));
configurationInitialised = true;
} else {
// there was a problem loading the configuration file
log.log("\nSystem configuration file has errors - please check your configuration");
}
}
} else {
// 'user' configuration file exists
// can we load the configuration file without error?
if (loadConfigFile(userConfigFilePath)) {
// configuration file loaded without error
log.log("Configuration file successfully loaded");
// Set 'applicableConfigFilePath' to equal the 'config' we loaded
applicableConfigFilePath = userConfigFilePath;
configurationInitialised = true;
} else {
// there was a problem loading the configuration file
log.log("\nConfiguration file has errors - please check your configuration");
}
}
// Advise the user path that we will use for the application state data
if (canFind(applicableConfigFilePath, configDirName)) {
log.vlog("Using 'user' configuration path for application state data: ", configDirName);
} else {
if (canFind(applicableConfigFilePath, systemConfigDirName)) {
log.vlog("Using 'system' configuration path for application state data: ", systemConfigDirName);
}
}
// return if the configuration was initialised
return configurationInitialised;
}
// Create a backup of the 'config' file if it does not exist
void createBackupConfigFile() {
if (!getValueBool("dry_run")) {
// Is there a backup of the config file if the config file exists?
if (exists(applicableConfigFilePath)) {
log.vdebug("Creating a backup of the applicable config file");
// create backup copy of current config file
std.file.copy(applicableConfigFilePath, configBackupFile);
// File Copy should only be readable by the user who created it - 0600 permissions needed
configBackupFile.setAttributes(convertedPermissionValue);
}
} else {
// --dry-run scenario ... technically we should not be making any local file changes .......
log.log("DRY RUN: Not creating backup config file as --dry-run has been used");
}
}
// Return a given string value based on the provided key
string getValueString(string key) {
auto p = key in stringValues;
if (p) {
return *p;
} else {
throw new Exception("Missing config value: " ~ key);
}
}
// Return a given long value based on the provided key
long getValueLong(string key) {
auto p = key in longValues;
if (p) {
return *p;
} else {
throw new Exception("Missing config value: " ~ key);
}
}
// Return a given bool value based on the provided key
bool getValueBool(string key) {
auto p = key in boolValues;
if (p) {
return *p;
} else {
throw new Exception("Missing config value: " ~ key);
}
}
// Set a given string value based on the provided key
void setValueString(string key, string value) {
stringValues[key] = value;
}
// Set a given long value based on the provided key
void setValueLong(string key, long value) {
longValues[key] = value;
}
// Set a given long value based on the provided key
void setValueBool(string key, bool value) {
boolValues[key] = value;
}
// Configure the directory octal permission value
void configureRequiredDirectoryPermisions() {
// return the directory permission mode required
// - return octal!defaultDirectoryPermissionMode; ... cant be used .. which is odd
// Error: variable defaultDirectoryPermissionMode cannot be read at compile time
if (getValueLong("sync_dir_permissions") != defaultDirectoryPermissionMode) {
// return user configured permissions as octal integer
string valueToConvert = to!string(getValueLong("sync_dir_permissions"));
auto convertedValue = parse!long(valueToConvert, 8);
configuredDirectoryPermissionMode = to!int(convertedValue);
} else {
// return default as octal integer
string valueToConvert = to!string(defaultDirectoryPermissionMode);
auto convertedValue = parse!long(valueToConvert, 8);
configuredDirectoryPermissionMode = to!int(convertedValue);
}
}
// Configure the file octal permission value
void configureRequiredFilePermisions() {
// return the file permission mode required
// - return octal!defaultFilePermissionMode; ... cant be used .. which is odd
// Error: variable defaultFilePermissionMode cannot be read at compile time
if (getValueLong("sync_file_permissions") != defaultFilePermissionMode) {
// return user configured permissions as octal integer
string valueToConvert = to!string(getValueLong("sync_file_permissions"));
auto convertedValue = parse!long(valueToConvert, 8);
configuredFilePermissionMode = to!int(convertedValue);
} else {
// return default as octal integer
string valueToConvert = to!string(defaultFilePermissionMode);
auto convertedValue = parse!long(valueToConvert, 8);
configuredFilePermissionMode = to!int(convertedValue);
}
}
// Read the configuredDirectoryPermissionMode and return
int returnRequiredDirectoryPermisions() {
if (configuredDirectoryPermissionMode == 0) {
// the configured value is zero, this means that directories would get
// values of d---------
configureRequiredDirectoryPermisions();
}
return configuredDirectoryPermissionMode;
}
// Read the configuredFilePermissionMode and return
int returnRequiredFilePermisions() {
if (configuredFilePermissionMode == 0) {
// the configured value is zero
configureRequiredFilePermisions();
}
return configuredFilePermissionMode;
}
// Load a configuration file from the provided filename
private bool loadConfigFile(string filename) {
// configure function variables
try {
log.log("Reading configuration file: ", filename);
readText(filename);
} catch (std.file.FileException e) {
// Unable to access required file
log.error("ERROR: Unable to access ", e.msg);
// Use exit scopes to shutdown API
return false;
}
// We were able to readText the config file - so, we should be able to open and read it
auto file = File(filename, "r");
string lineBuffer;
// configure scopes
// - failure
scope(failure) {
// close file if open
if (file.isOpen()){
// close open file
file.close();
}
}
// - exit
scope(exit) {
// close file if open
if (file.isOpen()){
// close open file
file.close();
}
}
// read file line by line
auto range = file.byLine();
foreach (line; range) {
lineBuffer = stripLeft(line).to!string;
if (lineBuffer.length == 0 || lineBuffer[0] == ';' || lineBuffer[0] == '#') continue;
auto c = lineBuffer.matchFirst(configRegex);
if (!c.empty) {
c.popFront(); // skip the whole match
string key = c.front.dup;
auto p = key in boolValues;
if (p) {
c.popFront();
// only accept "true" as true value. TODO Should we support other formats?
setValueBool(key, c.front.dup == "true" ? true : false);
// skip_dotfiles tracking for change
if (key == "skip_dotfiles") {
configFileSkipDotfiles = true;
}
// skip_symlinks tracking for change
if (key == "skip_symlinks") {
configFileSkipSymbolicLinks = true;
}
// sync_business_shared_items tracking for change
if (key == "sync_business_shared_items") {
configFileSyncBusinessSharedItems = true;
}
} else {
auto pp = key in stringValues;
if (pp) {
c.popFront();
setValueString(key, c.front.dup);
// detect need for --resync for these:
// --syncdir ARG
// --skip-file ARG
// --skip-dir ARG
// sync_dir
if (key == "sync_dir") {
// configure a temp variable
string tempSyncDirValue = c.front.dup;
// is this empty ?
if (!strip(tempSyncDirValue).empty) {
configFileSyncDir = tempSyncDirValue;
} else {
// sync_dir cannot be empty
log.error("Invalid value for key in config file: ", key);
log.error("sync_dir in config file cannot be empty - this is a fatal error and must be corrected");
exit(EXIT_FAILURE);
}
}
// skip_file
if (key == "skip_file") {
// Handle multiple entries of skip_file
if (configFileSkipFile.empty) {
// currently no entry exists
configFileSkipFile = c.front.dup;
} else {
// add to existing entry
configFileSkipFile = configFileSkipFile ~ "|" ~ to!string(c.front.dup);
setValueString("skip_file", configFileSkipFile);
}
}
// skip_dir
if (key == "skip_dir") {
// Handle multiple entries of skip_dir
if (configFileSkipDir.empty) {
// currently no entry exists
configFileSkipDir = c.front.dup;
} else {
// add to existing entry
configFileSkipDir = configFileSkipDir ~ "|" ~ to!string(c.front.dup);
setValueString("skip_dir", configFileSkipDir);
}
}
// --single-directory Strip quotation marks from path
// This is an issue when using ONEDRIVE_SINGLE_DIRECTORY with Docker
if (key == "single_directory") {
// Strip quotation marks from provided path
string configSingleDirectory = strip(to!string(c.front.dup), "\"");
setValueString("single_directory", configSingleDirectory);
}
// Azure AD Configuration
if (key == "azure_ad_endpoint") {
string azureConfigValue = strip(c.front.dup);
switch(azureConfigValue) {
case "":
log.log("Using config option for Global Azure AD Endpoints");
break;
case "USL4":
log.log("Using config option for Azure AD for US Government Endpoints");
break;
case "USL5":
log.log("Using config option for Azure AD for US Government Endpoints (DOD)");
break;
case "DE":
log.log("Using config option for Azure AD Germany");
break;
case "CN":
log.log("Using config option for Azure AD China operated by 21Vianet");
break;
// Default - all other entries
default:
log.log("Unknown Azure AD Endpoint - using Global Azure AD Endpoints");
}
}
// Application ID
if (key == "application_id") {
// This key cannot be empty
string tempApplicationId = strip(c.front.dup);
if (tempApplicationId.empty) {
log.log("Invalid value for key in config file - using default value: ", key);
log.vdebug("application_id in config file cannot be empty - using default application_id");
setValueString("application_id", defaultApplicationId);
} else {
setValueString("application_id", tempApplicationId);
}
}
// Drive ID
if (key == "drive_id") {
// This key cannot be empty
string tempApplicationId = strip(c.front.dup);
if (tempApplicationId.empty) {
log.error("Invalid value for key in config file: ", key);
log.error("drive_id in config file cannot be empty - this is a fatal error and must be corrected");
exit(EXIT_FAILURE);
} else {
setValueString("drive_id", tempApplicationId);
configFileDriveId = tempApplicationId;
}
}
// Log Directory
if (key == "log_dir") {
// This key cannot be empty
string tempLogDir = strip(c.front.dup);
if (tempLogDir.empty) {
log.log("Invalid value for key in config file - using default value: ", key);
log.vdebug("log_dir in config file cannot be empty - using default log_dir");
setValueString("log_dir", defaultLogFileDir);
} else {
setValueString("log_dir", tempLogDir);
}
}
} else {
auto ppp = key in longValues;
if (ppp) {
c.popFront();
ulong thisConfigValue;
// Can this value actually be converted to an integer?
try {
thisConfigValue = to!long(c.front.dup);
} catch (std.conv.ConvException) {
log.log("Invalid value for key in config file: ", key);
return false;
}
setValueLong(key, thisConfigValue);
// if key is 'monitor_interval' the value must be 300 or greater
if (key == "monitor_interval") {
// temp value
ulong tempValue = thisConfigValue;
// the temp value needs to be greater than 300
if (tempValue < 300) {
log.log("Invalid value for key in config file - using default value: ", key);
tempValue = 300;
}
setValueLong("monitor_interval", to!long(tempValue));
}
// if key is 'monitor_fullscan_frequency' the value must be 12 or greater
if (key == "monitor_fullscan_frequency") {
// temp value
ulong tempValue = thisConfigValue;
// the temp value needs to be greater than 12
if (tempValue < 12) {
// If this is not set to zero (0) then we are not disabling 'monitor_fullscan_frequency'
if (tempValue != 0) {
// invalid value
log.log("Invalid value for key in config file - using default value: ", key);
tempValue = 12;
}
}
setValueLong("monitor_fullscan_frequency", to!long(tempValue));
}
// if key is 'space_reservation' we have to calculate MB -> bytes
if (key == "space_reservation") {
// temp value
ulong tempValue = thisConfigValue;
// a value of 0 needs to be made at least 1MB ..
if (tempValue == 0) {
log.log("Invalid value for key in config file - using 1MB: ", key);
tempValue = 1;
}
setValueLong("space_reservation", to!long(tempValue * 2^^20));
}
// if key is 'ip_protocol_version' this has to be a value of 0 or 1 or 2 .. nothing else
if (key == "ip_protocol_version") {
// temp value
ulong tempValue = thisConfigValue;
// If greater than 2, set to default
if (tempValue > 2) {
log.log("Invalid value for key in config file - using default value: ", key);
// Set to default of 0
tempValue = 0;
}
setValueLong("ip_protocol_version", to!long(tempValue));
}
} else {
log.log("Unknown key in config file: ", key);
bool ignore_depreciation = false;
// min_notify_changes has been depreciated
if (key == "min_notify_changes") {
log.log("\nThe option 'min_notify_changes' has been depreciated and will be ignored. Please read the updated documentation and update your client configuration.");
writeln();
ignore_depreciation = true;
}
// force_http_2 has been depreciated
if (key == "force_http_2") {
log.log("\nThe option 'force_http_2' has been depreciated and will be ignored. Please read the updated documentation and update your client configuration.");
writeln();
ignore_depreciation = true;
}
// Application configuration update required for Business Shared Folders
if (key == "sync_business_shared_folders") {
log.log("\nThe process for synchronising Microsoft OneDrive Business Shared Folders has changed.");
log.log("Please review the revised documentation on how to configure this application feature. You must update your client configuration and make any necessary online adjustments accordingly.");
writeln();
}
// Return false
return ignore_depreciation;
}
}
}
} else {
log.log("Malformed config line: ", lineBuffer);
return false;
}
}
// Close the file access
file.close();
// Free object and memory
object.destroy(file);
object.destroy(range);
return true;
}
// Update the application configuration based on CLI passed in parameters
void updateFromArgs(string[] cliArgs) {
// Add additional options that are NOT configurable via config file
stringValues["create_directory"] = "";
stringValues["create_share_link"] = "";
stringValues["destination_directory"] = "";
stringValues["get_file_link"] = "";
stringValues["modified_by"] = "";
stringValues["sharepoint_library_name"] = "";
stringValues["remove_directory"] = "";
stringValues["single_directory"] = "";
stringValues["source_directory"] = "";
stringValues["auth_files"] = "";
stringValues["auth_response"] = "";
boolValues["display_config"] = false;
boolValues["display_sync_status"] = false;
boolValues["display_quota"] = false;
boolValues["print_token"] = false;
boolValues["logout"] = false;
boolValues["reauth"] = false;
boolValues["monitor"] = false;
boolValues["synchronize"] = false;
boolValues["force"] = false;
boolValues["list_business_shared_items"] = false;
boolValues["force_sync"] = false;
boolValues["with_editing_perms"] = false;
// Application Startup option validation
try {
string tmpStr;
bool tmpBol;
long tmpVerb;
// duplicated from main.d to get full help output!
auto opt = getopt(
cliArgs,
std.getopt.config.bundling,
std.getopt.config.caseSensitive,
"auth-files",