From a8974ef0ae0e0f3c7ef84d41911c9ad29a22c8d7 Mon Sep 17 00:00:00 2001 From: Mark Banner Date: Wed, 15 Mar 2017 12:11:20 +0000 Subject: [PATCH] (chore): Add Flake8 to lint tests, change some files to have .py extensions and get flake8 passing. --- .gitignore | 1 + .travis.yml | 3 ++ Makefile | 20 ++++++++--- bin/build-scripts/{css_to_js => css_to_js.py} | 0 bin/build-scripts/fixup_panel_js | 18 ---------- bin/build-scripts/fixup_panel_js.py | 23 ++++++++++++ ...ut_version => sub_rdf_checkout_version.py} | 0 .../{update_manifest => update_manifest.py} | 1 - .../{write_build_time => write_build_time.py} | 0 bin/load_test_exercise.py | 36 ++++++++++++++----- bin/load_test_exercise_images.py | 2 ++ bin/require.pip | 1 + package.json | 1 + setup.cfg | 5 +++ 14 files changed, 78 insertions(+), 33 deletions(-) rename bin/build-scripts/{css_to_js => css_to_js.py} (100%) delete mode 100755 bin/build-scripts/fixup_panel_js create mode 100755 bin/build-scripts/fixup_panel_js.py rename bin/build-scripts/{sub_rdf_checkout_version => sub_rdf_checkout_version.py} (100%) rename bin/build-scripts/{update_manifest => update_manifest.py} (99%) rename bin/build-scripts/{write_build_time => write_build_time.py} (100%) create mode 100644 bin/require.pip create mode 100644 setup.cfg diff --git a/.gitignore b/.gitignore index d717299e83..edf994dec5 100644 --- a/.gitignore +++ b/.gitignore @@ -10,6 +10,7 @@ dist-production addon.log cookie-jar.txt .env.production +.venv /scratch /data *.log diff --git a/.travis.yml b/.travis.yml index d168951a60..b316077595 100644 --- a/.travis.yml +++ b/.travis.yml @@ -23,6 +23,9 @@ before_install: - "export DISPLAY=:99.0" - "/sbin/start-stop-daemon --start --quiet --pidfile /tmp/custom_xvfb_99.pid --make-pidfile --background --exec /usr/bin/Xvfb -- :99 -ac -screen 0 1280x1024x16 -extension RANDR" +install: + - pip install --user virtualenv + before_script: - export JPM_FIREFOX_BINARY=`which firefox` - firefox -v diff --git a/Makefile b/Makefile index 7298e56281..8a92bbe056 100644 --- a/Makefile +++ b/Makefile @@ -2,6 +2,7 @@ PATH := ./node_modules/.bin:./bin/:$(PATH) SHELL := /bin/bash BABEL := babel --retain-lines RSYNC := rsync --archive +VENV := .venv .DEFAULT_GOAL := help # Sets $(PAGESHOT_BACKEND) to http://localhost:10080 only if it isn't set PAGESHOT_BACKEND ?= http://localhost:10080 @@ -54,7 +55,7 @@ build/server/static/js/%.js: build/static/js/%.js build/%.js: %.js @mkdir -p $(@D) - $(BABEL) $< | ./bin/build-scripts/fixup_panel_js > $@ + $(BABEL) $< | ./bin/build-scripts/fixup_panel_js.py > $@ build/server/%.js: server/src/%.js @mkdir -p $(@D) @@ -103,6 +104,14 @@ else FIND_COMMAND := find -E $(GIT_EXPORT_DIR) endif +$(VENV): bin/require.pip + virtualenv -p python2.7 $(VENV) + . $(VENV)/bin/activate && pip install -r bin/require.pip + +.PHONY: flake8 +flake8: $(VENV) + $(VENV)/bin/flake8 . + .PHONY: export_addon export_addon: addon $(FIND_COMMAND) -type f ! -regex \ @@ -132,10 +141,10 @@ addon_locales: ./bin/build-scripts/pontoon-to-webext.js --dest addon/webextension/_locales addon/install.rdf: addon/install.rdf.template - ./bin/build-scripts/update_manifest $< $@ + ./bin/build-scripts/update_manifest.py $< $@ addon/webextension/manifest.json: addon/webextension/manifest.json.template build/.backend.txt package.json - ./bin/build-scripts/update_manifest $< $@ + ./bin/build-scripts/update_manifest.py $< $@ addon/webextension/build/shot.js: shared/shot.js @mkdir -p $(@D) @@ -143,7 +152,7 @@ addon/webextension/build/shot.js: shared/shot.js addon/webextension/build/inlineSelectionCss.js: build/server/static/css/inline-selection.css @mkdir -p $(@D) - ./bin/build-scripts/css_to_js inlineSelectionCss $< > $@ + ./bin/build-scripts/css_to_js.py inlineSelectionCss $< > $@ addon/webextension/build/raven.js: $(raven_source) @mkdir -p $(@D) @@ -193,7 +202,7 @@ build/server/static/js/creating-bundle.js: $(creating_dependencies) # anyway: build/server/build-time.js: homepage $(server_dest) $(shared_server_dest) $(sass_server_dest) $(imgs_server_dest) $(static_js_dest) $(patsubst server/db-patches/%,build/server/db-patches/%,$(wildcard server/db-patches/*)) @mkdir -p $(@D) - ./bin/build-scripts/write_build_time > build/server/build-time.js + ./bin/build-scripts/write_build_time.py > build/server/build-time.js .PHONY: server server: npm build/server/build-time.js build/server/package.json build/server/static/js/shot-bundle.js build/server/static/js/homepage-bundle.js build/server/static/js/metrics-bundle.js build/server/static/js/shotindex-bundle.js build/server/static/js/leave-bundle.js build/server/static/js/creating-bundle.js @@ -244,6 +253,7 @@ all: addon server .PHONY: clean clean: rm -rf build/ addon/webextension/build/ addon/webextension/manifest.json addon/webextension/_locales/ + rm -rf $(VENV) .PHONY: help help: diff --git a/bin/build-scripts/css_to_js b/bin/build-scripts/css_to_js.py similarity index 100% rename from bin/build-scripts/css_to_js rename to bin/build-scripts/css_to_js.py diff --git a/bin/build-scripts/fixup_panel_js b/bin/build-scripts/fixup_panel_js deleted file mode 100755 index f58db8a10a..0000000000 --- a/bin/build-scripts/fixup_panel_js +++ /dev/null @@ -1,18 +0,0 @@ -#!/usr/bin/env python - -import sys -orig_content = content = sys.stdin.read() -for line, repl in [ - ("Function.prototype", "Object.getPrototypeOf(function () {})"), - ("setTimeout(drainQueue, 0)", "setTimeout(function () {drainQueue()}, 0)"), - ("setTimeout(cleanUpNextTick)", "setTimeout(function () {cleanUpNextTick()})"), - ("element.setAttribute(eventName, \x27return;\x27);", "//element.setAttribute(eventName, \x27return;\x27);"), - ("isSupported = typeof element[eventName] === \x27function\x27;", "isSupported = true // disabled check"), -]: - if not content.count(line): - pass #sys.stderr.write("Warning: could not find match in panel-bundle: %r\n" % line) - else: - content = content.replace(line, repl) -if len(orig_content) != len(content): - sys.stderr.write(" Rewrote from %i to %i bytes\n" % (len(orig_content), len(content))) -sys.stdout.write(content) diff --git a/bin/build-scripts/fixup_panel_js.py b/bin/build-scripts/fixup_panel_js.py new file mode 100755 index 0000000000..18ac2994f4 --- /dev/null +++ b/bin/build-scripts/fixup_panel_js.py @@ -0,0 +1,23 @@ +#!/usr/bin/env python + +import sys +orig_content = content = sys.stdin.read() +for line, repl in [( + "Function.prototype", "Object.getPrototypeOf(function () {})" +), ( + "setTimeout(drainQueue, 0)", "setTimeout(function () {drainQueue()}, 0)" +), ( + "setTimeout(cleanUpNextTick)", "setTimeout(function () {cleanUpNextTick()})" +), ( + "element.setAttribute(eventName, \x27return;\x27);", "//element.setAttribute(eventName, \x27return;\x27);" +), ( + "isSupported = typeof element[eventName] === \x27function\x27;", "isSupported = true // disabled check" +)]: + if not content.count(line): + # sys.stderr.write("Warning: could not find match in panel-bundle: %r\n" % line) + pass + else: + content = content.replace(line, repl) +if len(orig_content) != len(content): + sys.stderr.write(" Rewrote from %i to %i bytes\n" % (len(orig_content), len(content))) +sys.stdout.write(content) diff --git a/bin/build-scripts/sub_rdf_checkout_version b/bin/build-scripts/sub_rdf_checkout_version.py similarity index 100% rename from bin/build-scripts/sub_rdf_checkout_version rename to bin/build-scripts/sub_rdf_checkout_version.py diff --git a/bin/build-scripts/update_manifest b/bin/build-scripts/update_manifest.py similarity index 99% rename from bin/build-scripts/update_manifest rename to bin/build-scripts/update_manifest.py index 45ceed9126..9451bd8882 100755 --- a/bin/build-scripts/update_manifest +++ b/bin/build-scripts/update_manifest.py @@ -6,7 +6,6 @@ import sys import os -import re import time import json import calendar diff --git a/bin/build-scripts/write_build_time b/bin/build-scripts/write_build_time.py similarity index 100% rename from bin/build-scripts/write_build_time rename to bin/build-scripts/write_build_time.py diff --git a/bin/load_test_exercise.py b/bin/load_test_exercise.py index 5895349e12..c2d575a9fb 100755 --- a/bin/load_test_exercise.py +++ b/bin/load_test_exercise.py @@ -1,4 +1,5 @@ #!/usr/bin/env python + import requests import argparse from urlparse import urljoin @@ -22,8 +23,8 @@ help='Search /shots N times with randomish queries') parser.add_argument('--times', metavar='M', type=int, default=1, help='Do everything M times (i.e., do each action N x M times)') -## FIXME: no option for /redirect -## FIXME: no option for /event +# FIXME: no option for /redirect +# FIXME: no option for /event example_images = {} execfile(os.path.join(os.path.dirname(__file__), "load_test_exercise_images.py"), example_images) @@ -35,19 +36,23 @@ session = requests.Session() + def make_device_info(): return dict( addonVersion='0.1.2014test', platform='test', ) + def make_uuid(): return str(uuid.uuid1()).replace("-", "") + deviceInfo = make_device_info() deviceId = make_uuid() secret = make_uuid() + def login(): resp = session.post( urljoin(backend, "/api/login"), @@ -58,12 +63,14 @@ def login(): data=dict(deviceId=deviceId, secret=secret, deviceInfo=json.dumps(deviceInfo))) resp.raise_for_status() + def delete_account(): resp = session.post( urljoin(backend, "/leave-page-shot/leave"), json={}) resp.raise_for_status() + def create_shot(): shot_id = make_uuid() + "/test.com" shot_url = urljoin(backend, shot_id) @@ -75,21 +82,25 @@ def create_shot(): resp.raise_for_status() return shot_url + def read_shot(url): - ## FIXME: should get at least the clip image subresource itself + # FIXME: should get at least the clip image subresource itself resp = session.get(url) resp.raise_for_status() + def read_my_shots(): resp = session.get(urljoin(backend, "/shots")) resp.raise_for_status() + def search_shots(q=None): if q is None: q = make_search_query() resp = session.get(urljoin(backend, "/shots"), params={"q": q}) resp.raise_for_status() + def make_example_shot(): image = random.choice(example_images) text = [] @@ -100,12 +111,12 @@ def make_example_shot(): deviceId=deviceId, url="http://test.com/?" + make_uuid(), docTitle="Load test page", - createdDate=int(time.time()*1000), + createdDate=int(time.time() * 1000), favicon=None, siteName="test site", clips={ make_uuid(): dict( - createdDate=int(time.time()*1000), + createdDate=int(time.time() * 1000), sortOrder=100, image=dict( url=image["url"], @@ -114,8 +125,8 @@ def make_example_shot(): location=dict( top=100, left=100, - bottom=100+image["height"], - right=100+image["width"], + bottom=100 + image["height"], + right=100 + image["width"], ), dimensions=dict( x=image["width"], @@ -126,11 +137,15 @@ def make_example_shot(): }, ) + def make_search_query(): return random.choice(search_strings) + text_strings = """ -Example strings like apple orange banana some stuff like whatever and whoever and bucket blanket funky etc keyboard screen house window tree leaf leaves feather feathers +Example strings like apple orange banana some stuff like whatever and whoever +and bucket blanket funky etc keyboard screen house window tree leaf leaves +feather feathers """.split() search_strings = """ @@ -139,6 +154,7 @@ def make_search_query(): login_happened = False + def run(): global login_happened shot_urls = [] @@ -175,15 +191,17 @@ def run(): total -= 1 break + def main(): try: for i in range(args.times): run() - print "Finished run %i/%i" % (i+1, args.times) + print "Finished run %i/%i" % (i + 1, args.times) except KeyboardInterrupt: print "Early abort" print "Deleting account" delete_account() + if __name__ == "__main__": main() diff --git a/bin/load_test_exercise_images.py b/bin/load_test_exercise_images.py index 92309a7828..44eaa356f0 100644 --- a/bin/load_test_exercise_images.py +++ b/bin/load_test_exercise_images.py @@ -1,3 +1,5 @@ +# flake8: noqa + example_images = [ dict( source="https://upload.wikimedia.org/wikipedia/commons/a/a9/Example.jpg", diff --git a/bin/require.pip b/bin/require.pip new file mode 100644 index 0000000000..df247afa73 --- /dev/null +++ b/bin/require.pip @@ -0,0 +1 @@ +flake8==3.3.0 diff --git a/package.json b/package.json index c241734386..5d2d15a179 100644 --- a/package.json +++ b/package.json @@ -71,6 +71,7 @@ "scripts": { "lint": "npm-run-all lint:*", "lint:addon": "make zip && addons-linter ./build/pageshot.zip -o text", + "lint:flake8": "make flake8", "lint:js": "eslint .", "lint:sass": "sass-lint -v -q", "postlint": "nsp check -o summary", diff --git a/setup.cfg b/setup.cfg new file mode 100644 index 0000000000..0a2a37cdd2 --- /dev/null +++ b/setup.cfg @@ -0,0 +1,5 @@ +[flake8] +exclude = .venv,built,dist,node_modules +max-line-length = 120 +# E402 module level import not at top of file (test files need to set up paths) +ignore=E402