forked from thewca/tnoodle
-
Notifications
You must be signed in to change notification settings - Fork 0
/
tmt
executable file
·1220 lines (1045 loc) · 45.6 KB
/
tmt
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
#!/usr/bin/env python3
from os.path import exists, dirname, join, isdir, split, abspath, relpath, basename
import sys, os, imp, re, subprocess, shutil, shlex
import zipfile
import tempfile
import time
import math
import json
import datetime
import atexit
import tntdebug
javaEEProjectName = "javaee-api-6.0.jar"
TNOODLE_FORCE_VERSION = "TNOODLE_FORCE_VERSION"
def checkShellCmd(*args, **kwargs):
kwargs['assertSuccess'] = True
kwargs['shell'] = True
return runCmd(*args, **kwargs)
def runCmd(argv, showStatus=False, interactive=False, assertSuccess=False, shell=False):
if showStatus:
print(argv)
if not shell:
print(" ".join(argv))
if interactive:
p = subprocess.Popen(argv)
p.wait()
return p.returncode
p = subprocess.Popen(argv, shell=shell, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
# This doesn't interleave stdout and stderr like it should, but at least
# it won't cause deadlock due to pipes filling up.
# The best thing to do would probably be to implement our own communicate
# that also dumps to the screen.
stdout, stderr = p.communicate()
stdout = stdout.decode()
stderr = stderr.decode()
if showStatus:
print(stdout)
print(stderr)
if assertSuccess:
assert p.returncode == 0
return p.returncode, stdout, stderr
def entryPoint():
return split(abspath(__file__))[0]
gitToolsDir = join(entryPoint(), 'git-tools')
sys.path.append(gitToolsDir)
from argparse import ArgumentParser, ArgumentDefaultsHelpFormatter
import github
import setupGitHooks
import setupWindowsCygwinSymlinking
from setupWindowsCygwinSymlinking import createSymlinkIfNotExistsOrStale, rmtree
import lint
# These are the kinds of files we replace all instances of "%%VERSION%%" with
# the current version.
TXT_FILE_EXTENSIONS = [ '.js' ]
RESOURCE_FOLDER = 'tnoodle_resources'
RESOURCE_FOLDER_SRC = 'src_%s' % RESOURCE_FOLDER
RESOURCE_FOLDER_BIN = 'bin_%s' % RESOURCE_FOLDER
PROJECT_FILE_NAME = 'tmtproject.py'
def dfs(node, path=(), exclude=set()):
assert node not in path, "Not a dag!"
visited = [ node, ]
path += ( node, )
for child in node.getDependencies():
if child in exclude:
continue
visited += dfs(child, path, exclude=exclude)
return visited
def rightPrune(nodes):
prunedNodes = []
for node in reversed(nodes):
if node not in prunedNodes:
prunedNodes.insert(0, node)
return prunedNodes
def isTopologicalSort(sortedNodes, exclude=set()):
for i, node in enumerate(sortedNodes):
for child in node.getDependencies():
if child in exclude:
continue
childIndex = sortedNodes.index(child)
assert childIndex != i
if childIndex < i:
return False
return True
def topologicalSort(projects, exclude=set()):
potentialRoots = projects[:]
for excluded in exclude:
if excluded in potentialRoots:
potentialRoots.remove(excluded)
for project in projects:
for dep in project.getDependencies():
if dep in potentialRoots:
potentialRoots.remove(dep)
class UltimateRoot(tmt.TmtProject):
def __init__(self, name, description):
# Note that we do *not* call our parent's constructer here,
# as we do not want to be added to the global list of projects
self.name = name
self.description = description
def compile(self):
pass
def dist(self):
assert False, "make dist without a -p PROJECTNAME makes no sense"
def run(self):
assert False, "make run without a -p PROJECTNAME makes no sense"
def getDependencies(self):
return potentialRoots
def check(self):
return
def __str__(self, timestamps=False):
return "all"
def clean(self):
for dep in self.getDependencies():
dep.clean()
ultimateRoot = UltimateRoot(name=None, description="")
visitedNodes = dfs(ultimateRoot, exclude=exclude)
assert visitedNodes[0] == ultimateRoot
visitedNodes = visitedNodes[1:]
visitedNodes = rightPrune(visitedNodes)
assert isTopologicalSort(visitedNodes, exclude=exclude)
return visitedNodes, ultimateRoot
class tmt:
def _cdEntryPoint(self):
os.chdir(entryPoint())
assert exists('README.md')
if os.environ.get(TNOODLE_FORCE_VERSION):
self.VERSION = os.environ[TNOODLE_FORCE_VERSION]
elif os.environ.get("TRAVIS"):
self.VERSION = os.environ["TRAVIS_COMMIT"]
else:
retVal, stdout, stderr = tmt.runCmd([ 'git', 'describe' ])
assert retVal == 0
self.VERSION = stdout.strip()
versionPrefix = "v"
assert self.VERSION.startswith(versionPrefix)
self.VERSION = self.VERSION[len(versionPrefix):]
# Check if our working tree is dirty (uncommitted changes).
retVal, stdout, stderr = self.runCmd([ "git", "ls-files", "-m", "-o", "-d", "--exclude-standard" ])
if stdout:
self.VERSION += "-%s" % int(time.time())
# Check that the git hooks folder is all set up
setupGitHooks.setupGitHooksIfNotSetup()
def doTextSubstitution(self, txt):
# This might be a pretty heavy hammer, but who in the world
# would legitimately want to use the string "%%VERSION%%"?
# Famous last words?
return txt.replace("%%VERSION%%", tmt.VERSION)
def notifyDist(self, filename):
sizeMB = 1.0 * os.path.getsize(filename) / 2**20
# ls -h ceils fractions, so we do the same thing here
sizeMB = math.ceil(sizeMB*10)/10
print('Successfully created %s (size %sM)' % ( os.path.abspath(filename), sizeMB ))
def _loadTmtProjects(self):
self._cdEntryPoint()
projectFiles = ( join(dir, PROJECT_FILE_NAME) for dir in os.listdir('.') )
for projectFile in sorted(filter(exists, projectFiles)):
projectName = dirname(projectFile)
imp.load_source('', projectFile)
assert dirname(projectFile) in tmt.TmtProject.projects
for projectName, project in sorted(tmt.TmtProject.projects.items()):
project.afterProjects()
sortedProjects, fakeRoot = topologicalSort(list(tmt.TmtProject.projects.values()))
# The key None maps to a fake project that (indirectly) depends on everything
tmt.TmtProject.projects[None] = fakeRoot
for project in reversed(sortedProjects):
project.configure()
def isWorkspaceClean(self, printWhy=False):
retVal, stdout, stderr = tmt.runCmd(['git', 'ls-files', '--other', '--exclude-standard'])
assert retVal == 0
if stdout != '':
sys.stderr.write("Untracked files found, please deal with them before releasing:\n")
sys.stderr.write(stdout)
return False
retVal, stdout, stderr = tmt.runCmd(['git', 'diff', '--name-only'])
assert retVal == 0
if stdout != '':
sys.stderr.write("Edited files found, please commit them before releasing:\n")
sys.stderr.write(stdout)
return False
return True
def exitIfWorkspaceNotClean(self):
if not self.isWorkspaceClean(printWhy=True):
sys.exit(1)
def assertCommitsPushed(self):
retVal, stdout, stderr = tmt.runCmd(['git', 'log', '@{u}..'], showStatus=True)
assert retVal == 0
assert stdout == ''
assert stderr == ''
def _main(self):
self.args = tmt.parser.parse_args()
self._loadTmtProjects()
self.args.func()
def _graph(self):
project = tmt.TmtProject.projects[tmt.args.project]
if tmt.args.file:
# TODO look at tmt.args.file and generate a dotty file or something?
assert False, "not yet implemented"
elif tmt.args.show_classpath:
# Look at all for all EclipseProject this project depends on,
# and add them to the classpath. We have to do this weirdness
# because project may not refer to an EclipseProject.
entities = set()
for p in project.getRecursiveDependenciesTopoSorted():
if issubclass(type(p), EclipseProject):
entities.update(p.getClasspathEntities())
classpath = EclipseProject.toClasspath(entities)
print(classpath)
else:
print(project.prettyDependencies())
def _pullRequest(self):
self.exitIfWorkspaceNotClean()
self.assertCommitsPushed()
ghApi = github.getApi(organization="thewca", repo="tnoodle")
tmt.args.project = None
tmt.args.no_recursive = False
self._make(command='check')
body = tmt.args.body
body += "\n\n*************\nPull request automatically opened by " + \
"./tmt pull-request\n./tmt make check passed\n"
ghApi.pullRequest(title=tmt.args.title, body=body)
def _describe(self):
print(self.VERSION)
def _make(self, projectName=None, command=None):
if projectName is None:
projectName = tmt.args.project
if command is None:
command = tmt.args.command
project = tmt.TmtProject.projects[projectName]
recurse = True
if command in [ 'run', 'dist' ]:
# It doesn't make sense to recurse with these commands
recurse = False
dist = False
if recurse:
projects = project.getRecursiveDependenciesTopoSorted()
else:
projects = [ project ]
# Build everything we need before testing.
if command == "check":
tmt._make(projectName=project.name, command='compile')
projects.reverse()
for project in projects:
commandFunc = getattr(project, command, None)
assert commandFunc, 'No %s target in %s' % ( command, project )
commandFunc()
def _printVersion(self):
print(self.VERSION)
def _i18n(self):
assert tmt.args.command == "check"
i18nDir = 'webscrambles/src/i18n'
args = [ 'bundle', 'exec', 'wca_i18n', '--verbose', join(i18nDir, 'en.yml') ] + list(tmt.glob(i18nDir, r'.*\.yml$'))
runCmd(args, assertSuccess=True, showStatus=True)
def _release(self):
tmt.releasing = True
# Urg, we have to set some arguments explicitly because
# we're not picking up the defaults from the parser_make.
tmt.args.main = None
tmt.args.args = ""
tmt.args.debug = None
tmt.args.profile = None
if not tmt.args.dry_run:
ghApi = github.getApi(organization="thewca", repo="tnoodle")
tmt.exitIfWorkspaceNotClean()
pullCommand = 'git pull'
print(pullCommand)
assert 0 == os.system(pullCommand)
pullCommand = 'git pull git@github.com:thewca/tnoodle.git'
print(pullCommand)
assert 0 == os.system(pullCommand)
assert exists('bower.json')
with open('bower.json') as bower_json_file:
parsed_json = json.load(bower_json_file)
tmt.VERSION = parsed_json['version']
assert tmt.VERSION
# The way tnoodle-android discovers its version is by running
# "./tmt version", which doesn't care about our python instance's
# tmt.VERSION variable. Here we force that next python instance to use
# *our* overriden VERSION variable. This is so gross.
os.environ[ TNOODLE_FORCE_VERSION ] = tmt.VERSION
tag = "v%s" % tmt.VERSION
# We want to compile before we tag, that way we don't have to
# clean up if the code won't compile. We also want to set
# tmt.VERSION before we compile, so that we generate the correct
# files.
wcaProjectName = 'wca'
tmt.args.project = wcaProjectName # yuck =(
# We can't run a dist until after we create the tag. This way
# the version number of the resulting file will be correct.
wcaProject = tmt.TmtProject.projects[wcaProjectName]
wcaProject.dist()
# Copy tnoodle.js to the root of our repository and commit it (if necessary).
tmt.TmtProject.projects['tnoodlejs'].dist()
tnoodlejsFile = "tnoodlejs/WebContent/tnoodlejs/tnoodle.js"
copyTNoodleJsCommand = "cp %s tnoodle.js" % tnoodlejsFile
assert 0 == os.system(copyTNoodleJsCommand)
if not self.isWorkspaceClean():
# The workspace may not be clean because tnoodle.js changed.
commitTNoodleJsCommand = "git commit tnoodle.js -m 'tnoodlejs updated to %s'" % ( tag )
print(commitTNoodleJsCommand)
if not tmt.args.dry_run:
assert 0 == os.system(commitTNoodleJsCommand)
# After committing that file, there should not be anything left uncommitted.
tmt.exitIfWorkspaceNotClean()
if not tmt.args.dry_run:
tnoodleAndroidProject = tmt.TmtProject.projects['tnoodle-android']
# TODO - cannot figure out how to get
# http://gradle.org/docs/current/userguide/signing_plugin.html working.
# Since we're going to be rewriting tnoodle anyways, lets just not bother
# keeping maven up to date.
# tnoodleAndroidProject.publishToMavenCentral()
print("Releasing TNoodle")
tagMessage = "version %s" % tmt.VERSION
tagCommand = "git tag -a %s -m '%s'" % (tag, tagMessage)
print(tagCommand)
if not tmt.args.dry_run:
assert 0 == os.system(tagCommand)
print("Successfully created tag %s" % tag)
baseWcaFileName = os.path.basename(wcaProject.distJarFile())
files = [
( baseWcaFileName, open(wcaProject.distJarFile(), 'rb').read() ),
]
if not tmt.args.dry_run:
response, uploadResponses = ghApi.createRelease(tag, files=files)
for pushTags in [ False, True ]:
# git push --tags seems to not do everything git push does
pushCommand = 'git push'
if pushTags:
pushCommand += ' --tags'
print(pushCommand)
if not tmt.args.dry_run:
assert 0 == os.system(pushCommand)
pushCommand += ' git@github.com:thewca/tnoodle.git master'
print(pushCommand)
if not tmt.args.dry_run:
assert 0 == os.system(pushCommand)
if not tmt.args.dry_run:
print("*"*80)
print("Release draft successfully created! Visit %s to complete release." % response['html_url'])
def __init__(self):
self.releasing = False
self.parser = ArgumentParser(
description='tmt, the TNoodle Make Tools',
formatter_class=ArgumentDefaultsHelpFormatter)
subparsers = self.parser.add_subparsers(help='Available commands', dest='subparser_name')
subparsers.required = True # Something changed in python 3 (see http://hg.python.org/cpython/rev/cab204a79e09), so dumb
desc = 'Make stuff'
parser_make = subparsers.add_parser(
'make',
help=desc, description=desc,
formatter_class=ArgumentDefaultsHelpFormatter)
parser_make.add_argument(
'--project', '-p',
default=None,
type=str,
help='the project to build')
parser_make.add_argument(
'command',
nargs='?',
choices=['compile', 'dist', 'run', 'check', 'clean'],
default='compile',
help='command!')
parser_make.add_argument(
'--args', '-a',
default='',
help='Command line arguments (only valid with the "run" command)')
parser_make.add_argument(
'--main', '-m',
default='',
help='Main class to run')
parser_make.add_argument(
'--debug', '-d',
choices=['jdb', 'attach'],
default=None,
help='Run with jdb or as an attachable vm (only valid with the "run" command)')
parser_make.add_argument(
'--profile',
choices=['samples', 'times'],
default=None,
help='Profile program (stats will be written to profile.hprof)')
parser_make.set_defaults(func=self._make)
desc = 'Print version string to use for builds'
parser_version = subparsers.add_parser(
'version',
help=desc, description=desc,
formatter_class=ArgumentDefaultsHelpFormatter)
parser_version.set_defaults(func=self._printVersion)
desc = 'Publish TNoodle to github'
parser_release = subparsers.add_parser(
'release',
help=desc, description=desc,
formatter_class=ArgumentDefaultsHelpFormatter)
parser_release.add_argument(
'--dry-run', '-d',
default=False, action='store_true',
help="Generate the release targets, but don't actually tag anything or upload anything. Good for debugging releasing.")
parser_release.set_defaults(func=self._release)
desc = 'Generate dependency graph'
parser_graph = subparsers.add_parser(
'graph',
help=desc, description=desc,
formatter_class=ArgumentDefaultsHelpFormatter)
parser_graph.add_argument(
'--file', '-f',
default='', type=str,
help='')
parser_graph.add_argument(
'--project', '-p',
default=None, type=str,
help='the project to build')
parser_graph.add_argument(
'--timestamps', '-t',
action="store_true", default=False,
help='Show project timestamps for debugging purposes')
parser_graph.add_argument(
'--descriptions', '-d',
action="store_true", default=False,
help='Show project descriptions')
parser_graph.add_argument(
'--no-prune', '-np',
action="store_true", default=False,
help='Do not prune duplicate projects')
parser_graph.add_argument(
'--show-classpath',
action="store_true", default=False,
help='Print java classpath required to run project')
parser_graph.set_defaults(func=self._graph)
desc = lint.desc()
parser_lint = subparsers.add_parser(
'lint',
help=desc, description=desc,
formatter_class=ArgumentDefaultsHelpFormatter)
lint.setupArgparser(parser_lint)
def lintIt():
lint.main(tmt.args)
parser_lint.set_defaults(func=lintIt)
desc = 'i18n tools'
parser_i18n = subparsers.add_parser(
'i18n',
help=desc, description=desc,
formatter_class=ArgumentDefaultsHelpFormatter)
parser_i18n.set_defaults(func=self._i18n)
parser_i18n.add_argument(
'command',
choices=['check'],
help='command!')
desc = "Print the version"
describe = subparsers.add_parser(
'describe',
help=desc, description=desc,
formatter_class=ArgumentDefaultsHelpFormatter)
describe.set_defaults(func=self._describe)
desc = "Run tests & submit pull request if successful"
pull_request = subparsers.add_parser(
'pull-request',
help=desc, description=desc,
formatter_class=ArgumentDefaultsHelpFormatter)
pull_request.add_argument(
'title',
default=None, type=str,
help="Title for this pull request")
pull_request.add_argument(
'body',
default=None, type=str,
help="Body for this pull request")
pull_request.set_defaults(func=self._pullRequest)
def memoize(self, func):
d = {}
evaluating = set()
def memoed(*args):
if not args in d:
assert not args in evaluating, "Memoized functions may not have cycles!"
evaluating.add(args)
d[args] = func(*args)
evaluating.remove(args)
return d[args]
return memoed
def projectName(self):
import traceback
projectName, projectFileName = traceback.extract_stack()[-2][0].split(os.sep)
assert isdir(projectName)
assert projectFileName == PROJECT_FILE_NAME
return projectName
def rmtree(self, dir):
rmtree(dir)
def createSymlinkIfNotExistsOrStale(self, target, name):
createSymlinkIfNotExistsOrStale(target, name)
def runCmd(self, argv, showStatus=False, interactive=False):
return runCmd(argv, showStatus, interactive)
def showCmd(self, argv):
return self.runCmd(argv, showStatus=True)
def timestamp(self, f):
"""
If f is a directory, returns the timestamp of the newest file
found anywhere under the given directory f.
If f is a file, simply returns the timestamp of the given file.
Empty/non existent directories have timestamp 0.
"""
if not exists(f):
return 0
if os.path.islink(f):
# We don't follow symbolic links because we don't want them
# to cause a recompile of any project that uses us.
return os.lstat(f).st_mtime
if not isdir(f):
try:
return os.lstat(f).st_mtime
except:
import traceback
traceback.print_exc()
import pdb; pdb.set_trace()
m = 0
for ff in os.listdir(f):
m = max(m, self.timestamp(join(f, ff)))
return m
def glob(self, folder, pattern, relativeTo=None):
matches = set()
pattern = re.compile(pattern)
if relativeTo is None:
relativeTo = '.'
for root, dirs, files in os.walk(folder):
matches |= set( relpath(join(root, f), relativeTo) for f in files if pattern.match(f) )
return matches
def java(self, main, classpath='', args=[], debug=None, profile=None, wait=True, captureOutput=False, maxMemory=None):
command = []
assert debug in [ None, 'jdb', 'attach' ]
if debug == 'jdb':
command.append('jdb')
# TODO - passing -ea to jdb doesn't work! I HATE JAVA
else:
command.append('java')
if debug == 'attach':
command.append('-agentlib:jdwp=transport=dt_socket,address=8000,server=y,suspend=n') # TODO - configurable port?
command.append('-ea') # TODO - should turn on assertions at runtime
if profile:
command.append('-Xrunhprof:cpu=%s,depth=30,file=%s' % ( profile, 'profile.hprof' ))
if classpath:
command.append('-classpath')
command.append(classpath)
if maxMemory:
command.append('-Xmx' + maxMemory)
command.append(main)
command += args
print(" ".join(command))
stdout = None
stderr = None
if captureOutput:
assert wait
stdout = subprocess.PIPE
stderr = subprocess.PIPE
p = subprocess.Popen(command, stdout=stdout, stderr=stderr)
if wait:
if captureOutput:
stdoutdata, stderrdata = p.communicate()
return ( p.returncode, stdoutdata, stderrdata )
else:
return p.wait()
else:
def killProcess():
if p.poll() is None:
# Child process is still running, lets kill it!
p.kill()
atexit.register(killProcess)
return p
tmt = tmt() # We only want one instance of Tmt
sys.modules['tmt'] = tmt
import collections
class CaseInsensitiveDict(collections.Mapping):
def __init__(self, d=None):
if d is None:
d = {}
self._d = d
self._s = dict((k.lower(), k) for k in d)
def __contains__(self, k):
return k.lower() in self._s
def __len__(self):
return len(self._s)
def __iter__(self):
return iter(self._s)
def __getitem__(self, k):
lowerK = None if k is None else k.lower()
return self._d[self._s[lowerK]]
def __setitem__(self, k, v):
lowerK = None if k is None else k.lower()
self._d[k] = v
self._s[lowerK] = k
def actual_key_case(self, k):
return self._s.get(k.lower())
class TmtProject(object):
projects = CaseInsensitiveDict()
def __init__(self, name, description):
self.name = name
self.description = description
assert not name in TmtProject.projects, name
TmtProject.projects[name] = self
def __str__(self, timestamps=False):
return self.name
def __repr__(self):
return "<%s name=%s>" % ( type(self), str(self) )
# The configure method of each project gets called in dependency order.
# That is, if a project B uses a project A,
# A.configure() will be called before B.configure().
def configure(self):
pass
# The afterProjects method gets called for each project after all the projects
# have been loaded, in no particular order.
def afterProjects(self):
pass
def needsCompiling(self):
return False
def compile(self):
assert False
def dist(self):
assert False
def run(self):
assert False
def clean(self):
assert False
def getDependencies(self):
assert False
def getRecursiveDependenciesTopoSorted(self, exclude=set()):
return topologicalSort([self], exclude=exclude)[0]
def prettyDependencies(self, level=0, printingLevels=None, seen=None):
if printingLevels == None:
assert level == 0
printingLevels = set([0])
else:
printingLevels = printingLevels.copy()
if seen == None:
assert level == 0
seen = set()
indent = ''
for i in range(level):
if i == level-1:
if not i in printingLevels:
indent += '`-- '
else:
indent += '|-- '
else:
if i in printingLevels:
indent += '| '
else:
indent += ' '
s = indent
if self.needsCompiling():
s += '* '
s += self.__str__(timestamps=tmt.args.timestamps)
if not tmt.args.no_prune and self in seen:
# If we're pruning, and we've seen this node before,
# then we don't recurse
s += ' (seen)\n'
else:
if tmt.args.descriptions and self.description:
s += ' - ' + self.description
s += '\n'
printingLevels.add(level)
dependencies = self.getDependencies()
for p in dependencies:
if p == dependencies[-1]:
printingLevels.remove(level)
s += p.prettyDependencies(level+1, printingLevels, seen)
seen.add(p)
return s
tmt.TmtProject = TmtProject
@tmt.memoize
def createJarDependency(jarFile):
return JarDependency(jarFile)
class JarDependency(TmtProject):
def __init__(self, jarFile):
self.jarFile = jarFile
TmtProject.__init__(self,
basename(jarFile),
description="")
def check(self):
return
def getDependencies(self):
return []
def compile(self):
pass
def clean(self):
pass
def __str__(self, timestamps=False):
s = self.name
if timestamps:
s += ' %s' % tmt.timestamp(self.jarFile)
return s
class EclipseProject(TmtProject):
from xml.sax import make_parser, saxutils
class EclipseClasspathHandler(saxutils.handler.ContentHandler):
def startDocument(self):
self.jarFiles = []
self.projects = []
def startElement(self, name, attrs):
if name == 'classpathentry':
kind = attrs.get('kind')
path = attrs.get('path')
if kind == 'output':
self.project.bin = join(self.project.name, path)
elif kind == 'src':
if path.startswith('/'):
self.projects.append(path[1:])
else:
assert path == 'src' or path == RESOURCE_FOLDER_SRC
elif kind == 'lib':
assert path.startswith('/')
self.jarFiles.append(abspath(path[1:]))
elif kind == 'con':
pass
else:
assert False, 'Unrecognized kind: %s' % kind
classpathParser = make_parser()
classpathHandler = EclipseClasspathHandler()
classpathParser.setContentHandler(classpathHandler)
def __init__(self, name, description, main=None, argv=None):
self.main = main
self.argv = argv
self.nonJavaSrcDeps = set()
self.javaFilesToIgnore = set()
self.nonJavaResourceDeps = set()
self.strictCompile = True
self.ignoredWarnings = []
# We don't need to be warned about possible fallthroughs in switch statements
self.ignoredWarnings += [ 'fallthrough' ]
# This check is dumb, it doesn't realize casting is sometimes
# done to force a different method to be called.
self.ignoredWarnings += [ 'cast' ]
TmtProject.__init__(self, name, description)
def getDependencies(self, includeCompileTimeOnlyDependencies=False):
return self.getJars(includeCompileTimeOnlyDependencies=includeCompileTimeOnlyDependencies) + self.projects
def getJars(self, includeCompileTimeOnlyDependencies=False):
jars = self.jars[:]
javaEEProject = TmtProject.projects[javaEEProjectName]
if includeCompileTimeOnlyDependencies:
if self.webContent:
# We only need this jar in order to compile web projects.
# Including it in our runtime classpath is bad, because it doesn't provide
# actual implmementations for the api it defines. A servlet
# container (such as winstone) provides the actual implementation.
jars.append(javaEEProject)
else:
if javaEEProject in jars:
jars.remove(javaEEProject)
return jars
def getClasspathEntities(self, includeResources=True, includeCompileTimeOnlyDependencies=False, includeSrc=False):
classpath = set()
classpath |= set(relpath(jar.jarFile) for jar in self.getJars(includeCompileTimeOnlyDependencies=includeCompileTimeOnlyDependencies))
classpath.add(self.bin)
if includeResources:
classpath.add(self.binResource)
# The src folder only needs to be in the classpath so
# jdb can discover the source code, and so compilation of stuff in
# src_tnoodle_resources/ can find things defined in src/
if includeSrc:
classpath.add(self.src)
# TODO - I hate the world
# Uncomment & replace the following with the path to your jre
# if you're trying to use jbd inside of a java library method.
#classpath.add('/usr/lib/jvm/java-1.6.0-openjdk-1.6.0.0.x86_64/src')
for project in self.projects:
classpath |= project.getClasspathEntities(includeResources=includeResources, includeCompileTimeOnlyDependencies=includeCompileTimeOnlyDependencies, includeSrc=includeSrc)
return classpath
@staticmethod
def toClasspath(entities):
separator = None
if setupWindowsCygwinSymlinking.windowsOrCygwin():
separator = ';'
else:
separator = ':'
return separator.join(sorted(entities))
def afterProjects(self):
dependencyFile = join(self.name, '.classpath')
assert exists(dependencyFile), dependencyFile
# TODO - this only works if we're single threaded
EclipseProject.classpathHandler.project = self
EclipseProject.classpathParser.parse(dependencyFile)
self.jars = [ createJarDependency(jar) for jar in EclipseProject.classpathHandler.jarFiles ]
self.projects = [ tmt.TmtProject.projects[p] for p in EclipseProject.classpathHandler.projects ]
self.distDir = join(self.name, 'dist')
self.src = join(self.name, 'src')
self.testDir = join(self.name, 'test')
self.srcResource = join(self.name, RESOURCE_FOLDER_SRC)
self.binResource = join(self.name, RESOURCE_FOLDER_BIN)
self.fullName = 'TNoodle-%s' % ( self.name )
webContent = join(self.name, "WebContent")
if isdir(webContent):
self.webContent = webContent
else:
self.webContent = None
if not isdir(self.bin):
os.makedirs(self.bin)
if not isdir(self.binResource):
os.makedirs(self.binResource)
def distJarFile(self, includeVersion=True, extension='jar'):
# This is a function rather than a static attribute because
# tmt.VERSION can change (see release target).
if includeVersion:
fileName = '%s-%s.%s' % ( self.fullName, tmt.VERSION, extension )
else:
fileName = '%s.%s' % ( self.fullName, extension )
return join(self.distDir, fileName)
def needsCompiling(self):
# Note that we assume that if the resources of a project we depend on change,
# we don't need to recompile. This is the way resources should behave.
if any( dep.needsCompiling() for dep in self.getDependencies() ):
return True
lastCompiled = tmt.timestamp(self.bin)
depsLastTouched = ( ( tmt.timestamp(dep), dep ) for dep in self.getClasspathEntities(includeResources=False, includeSrc=True) )
changedDeps = [ts_dep for ts_dep in depsLastTouched if ts_dep[0] > lastCompiled]
srcLastTouched = max(tmt.timestamp(self.src), tmt.timestamp(self.srcResource))
if self.webContent:
srcLastTouched = max(srcLastTouched, tmt.timestamp(self.webContent))
return lastCompiled <= srcLastTouched or len(changedDeps) > 0
def compile(self):
if not self.needsCompiling():
print("%s is up to date, not recompiling" % self.name)
return False
self._compile(self.src, self.bin)
self._compile(self.srcResource, self.binResource)
for src, bin, nonJavaDeps in ( ( self.srcResource, self.binResource, self.nonJavaResourceDeps ), ( self.src, self.bin, self.nonJavaSrcDeps ) ):
for nonJavaDep in nonJavaDeps:
srcPath = join(src, nonJavaDep)
binPath = join(bin, nonJavaDep)
print("Copying %s -> %s" % ( srcPath, binPath ))
if isdir(srcPath):
parentDir = dirname(dirname(binPath))
else:
parentDir = dirname(binPath)
if not isdir(parentDir):
os.makedirs(parentDir)
if binPath.endswith('/'):
binPath = binPath[:-1]
srcPath = abspath(srcPath)
srcPath = relpath(srcPath, parentDir)
os.symlink(srcPath, binPath)
createSymlinkIfNotExistsOrStale(os.path.relpath(self.binResource, self.bin), join(self.bin, RESOURCE_FOLDER))
return True
def innerCompile(self, src, tempBin, bin):
# omg shoot me now
pass
def _compile(self, src, bin):
head, tail = os.path.split(bin)
tempBin = join(head, "." + tail)
print('Compiling: %s' % src)
rmtree(bin)
if isdir(tempBin):
rmtree(tempBin)
os.makedirs(tempBin)
javaFiles = tmt.glob(src, r'.*\.java$')
javaFiles -= self.javaFilesToIgnore
if len(javaFiles) > 0:
args = [ 'javac' ]
# Generate all debugging information,
# it doesn't seem to be *that* much stuff.
args += [ '-g' ]
args += [ '-d', tempBin ]
if self.strictCompile:
# Enable all warnings
args += [ '-Xlint' ]
# Treat warnings as errors
args += [ '-Werror' ]
for ignoredWarning in self.ignoredWarnings:
args += [ '-Xlint:-%s' % ignoredWarning ]
# TODO - some of our library jars may include a Class-path
# For now, I want to just manually edit those jars, but if we