ci: make consistent with pymplate

- python 3.12 testing
- pytz is not necessary anymore
- use ruff
- move tests inside the package

.ci/release

@@ -21,15 +21,15 @@ import shutil
 
 is_ci = os.environ.get('CI') is not None
 
-def main():
+def main() -> None:
     import argparse
     p = argparse.ArgumentParser()
     p.add_argument('--test', action='store_true', help='use test pypi')
     args = p.parse_args()
 
     extra = []
     if args.test:
-        extra.extend(['--repository-url', 'https://test.pypi.org/legacy/'])
+        extra.extend(['--repository', 'testpypi'])
 
     root = Path(__file__).absolute().parent.parent
     os.chdir(root) # just in case
@@ -42,7 +42,7 @@ def main():
     if dist.exists():
         shutil.rmtree(dist)
 
-    check_call('python3 setup.py sdist bdist_wheel', shell=True)
+    check_call(['python3', '-m', 'build'])
 
     TP = 'TWINE_PASSWORD'
     password = os.environ.get(TP)

.ci/run

@@ -37,4 +37,4 @@ if ! command -v python3 &> /dev/null; then
 fi
 
 "$PY_BIN" -m pip install --user tox
-"$PY_BIN" -m tox "$@"
+"$PY_BIN" -m tox --parallel --parallel-live "$@"

.github/workflows/main.yml

@@ -22,9 +22,10 @@ on:
 jobs:
   build:
     strategy:
+      fail-fast: false
       matrix:
         platform: [ubuntu-latest, macos-latest, windows-latest]
-        python-version: ['3.8', '3.9', '3.10', '3.11']
+        python-version: ['3.8', '3.9', '3.10', '3.11', '3.12']
         # vvv just an example of excluding stuff from matrix
         # exclude: [{platform: macos-latest, python-version: '3.6'}]
 
@@ -33,9 +34,6 @@ jobs:
     steps:
     # ugh https://github.com/actions/toolkit/blob/main/docs/commands.md#path-manipulation
     - run: echo "$HOME/.local/bin" >> $GITHUB_PATH
-    - if: ${{ matrix.platform == 'macos-latest' && matrix.python-version == '3.11' }}
-      # hmm somehow only seems necessary for 3.11 on osx?? 
-      run: echo "$HOME/Library/Python/${{ matrix.python-version }}/bin" >> $GITHUB_PATH
 
     - uses: actions/setup-python@v4
       with:
@@ -44,6 +42,7 @@ jobs:
     - uses: actions/checkout@v3
       with:
         submodules: recursive
+        fetch-depth: 0  # nicer to have all git history when debugging/for tests
 
     - uses: mxschmitt/action-tmate@v3
       if: ${{ github.event_name == 'workflow_dispatch' && inputs.debug_enabled }}
@@ -79,12 +78,12 @@ jobs:
       if: github.event_name != 'pull_request' && github.event.ref == 'refs/heads/master'
       env:
         TWINE_PASSWORD: ${{ secrets.TWINE_PASSWORD_TEST }}
-      run: pip3 install --user wheel twine && .ci/release --test
+      run: pip3 install --user --upgrade build twine && .ci/release --test
 
     - name: 'release to pypi'
       # always deploy tags to release pypi
       # NOTE: release tags are guarded by on: push: tags on the top
       if: github.event_name != 'pull_request' && startsWith(github.event.ref, 'refs/tags')
       env:
         TWINE_PASSWORD: ${{ secrets.TWINE_PASSWORD }}
-      run: pip3 install --user wheel twine && .ci/release
+      run: pip3 install --user --upgrade build twine && .ci/release

.gitmodules

@@ -1,3 +1,3 @@
 [submodule "tests/data/kobo_notes"]
-	path = tests/data/kobo_notes
+	path = testdata/kobo_notes
 	url = https://github.com/CharlyMartin/kobo_notes

conftest.py

@@ -0,0 +1,38 @@
+# this is a hack to monkey patch pytest so it handles tests inside namespace packages without __init__.py properly
+# without it, pytest can't discover the package root for some reason
+# also see https://github.com/karlicoss/pytest_namespace_pkgs for more
+
+import pathlib
+from typing import Optional
+
+import _pytest.main
+import _pytest.pathlib
+
+# we consider all dirs in repo/ to be namespace packages
+root_dir = pathlib.Path(__file__).absolute().parent.resolve() / 'src'
+assert root_dir.exists(), root_dir
+
+# TODO assert it contains package name?? maybe get it via setuptools..
+
+namespace_pkg_dirs = [str(d) for d in root_dir.iterdir() if d.is_dir()]
+
+# resolve_package_path is called from _pytest.pathlib.import_path
+# takes a full abs path to the test file and needs to return the path to the 'root' package on the filesystem
+resolve_pkg_path_orig = _pytest.pathlib.resolve_package_path
+def resolve_package_path(path: pathlib.Path) -> Optional[pathlib.Path]:
+    result = path  # search from the test file upwards
+    for parent in result.parents:
+        if str(parent) in namespace_pkg_dirs:
+            return parent
+    raise RuntimeError("Couldn't determine path for ", path)
+_pytest.pathlib.resolve_package_path = resolve_package_path
+
+
+# without patching, the orig function returns just a package name for some reason
+# (I think it's used as a sort of fallback)
+# so we need to point it at the absolute path properly
+# not sure what are the consequences.. maybe it wouldn't be able to run against installed packages? not sure..
+search_pypath_orig = _pytest.main.search_pypath
+def search_pypath(module_name: str) -> str:
+    return str(root_dir)
+_pytest.main.search_pypath = search_pypath

pyproject.toml

@@ -0,0 +1,39 @@
+# see https://github.com/karlicoss/pymplate for up-to-date reference
+[project]
+dynamic = ["version"]  # version is managed by setuptools_scm
+name = "kobuddy"
+dependencies = [
+]
+
+## these need to be set if you're planning to upload to pypi
+description = "Backup and extract data from your Kobo reader"
+license = {file = "LICENSE.txt"}
+authors = [
+    {name = "Dima Gerasimov (@karlicoss)", email = "karlicoss@gmail.com"},
+]
+maintainers = [
+    {name = "Dima Gerasimov (@karlicoss)", email = "karlicoss@gmail.com"},
+]
+
+[project.urls]
+Homepage = "https://github.com/karlicoss/kobuddy"
+##
+
+[project.optional-dependencies]
+testing = [
+    "pytest",
+    "ruff",
+    "mypy",
+    "lxml",  # for mypy html coverage
+]
+
+[project.scripts]
+kobuddy = "kobuddy:__main__.main"
+
+[build-system]
+requires = ["setuptools", "setuptools-scm"]
+build-backend = "setuptools.build_meta"
+
+[tool.setuptools_scm]
+version_scheme = "python-simplified-semver"
+local_scheme = "dirty-tag"

ruff.toml

@@ -0,0 +1,25 @@
+ignore = [
+### too opinionated style checks
+    "E501",  # too long lines
+    "E702",  # Multiple statements on one line (semicolon)
+    "E731",  # assigning lambda instead of using def
+    "E741",  # Ambiguous variable name: `l`
+    "E742",  # Ambiguous class name: `O
+    "E401",  # Multiple imports on one line
+    "F403",  # import *` used; unable to detect undefined names
+###
+
+###
+    "E722",  # Do not use bare `except` ## Sometimes it's useful for defensive imports and that sort of thing..
+    "F811",  # Redefinition of unused  # this gets in the way of pytest fixtures (e.g. in cachew)
+
+## might be nice .. but later and I don't wanna make it strict
+    "E402",  # Module level import not at top of file
+
+### maybe consider these soon
+# sometimes it's useful to give a variable a name even if we don't use it as a documentation
+# on the other hand, often is a sign of error
+    "F841",  # Local variable `count` is assigned to but never used
+    "F401",  # imported but unused
+###
+]

src/kobuddy/__init__.py

@@ -13,15 +13,13 @@
 import struct
 import sqlite3
 from contextlib import contextmanager
-from datetime import datetime
+from datetime import datetime, timezone
 from pathlib import Path
 from tempfile import NamedTemporaryFile
 import typing
 from typing import (Dict, Iterator, List, NamedTuple, Optional, Sequence, Set,
                     Tuple, Union, Iterable, Any)
 
-import pytz
-
 from .common import get_logger, unwrap, cproperty, group_by_key, the, nullcontext, Res, sorted_res, split_res
 from .kobo_device import get_kobo_mountpoint
 from .sqlite import sqlite_connection
@@ -130,13 +128,13 @@ def _parse_utcdt(s: Optional[str]) -> Optional[datetime]:
     assert res is not None
 
     if res.tzinfo is None:
-        res = pytz.utc.localize(res)
+        res = res.replace(tzinfo=timezone.utc)
     return res
 
 
 # TODO not so sure about inheriting event..
 class Highlight(Event):
-    def __init__(self, row: Dict[str, str], book: Book):
+    def __init__(self, row: Dict[str, Any], book: Book) -> None:
         self.row = row
         self._book = book
 
@@ -256,7 +254,7 @@ def summary(self) -> str:
 class StartEvent(OtherEvent):
     @property
     def summary(self) -> str:
-        return f'started'
+        return 'started'
 
 class FinishedEvent(OtherEvent):
     def __init__(self, *args, time_spent_s: Optional[int]=None, **kwargs) -> None:
@@ -674,7 +672,7 @@ def consume(fmt):
             dts = []
             for _ in range(cnt):
                 ts, = consume('>5xI')
-                dt = pytz.utc.localize(datetime.utcfromtimestamp(ts))
+                dt = datetime.fromtimestamp(ts, tz=timezone.utc)
                 dts.append(dt)
             part_data = dts
         elif name == b'ViewType':
@@ -929,7 +927,7 @@ def kkey(e):
 
         k = e.dt
         if k.tzinfo is None:
-            k = k.replace(tzinfo=pytz.utc)
+            k = k.replace(tzinfo=timezone.utc)
         return (k, cls_order)
     return list(sorted_res(iter_events(**kwargs), key=kkey))
 

tests/test_kobo.py → src/kobuddy/tests/test_kobo.py

@@ -1,12 +1,14 @@
-from datetime import datetime
+from datetime import datetime, timezone
 from pathlib import Path
-import pytz
 
 import kobuddy
 
-def get_test_db():
-    # db = Path(__file__).absolute().parent.parent / 'KoboShelfes' / 'KoboReader.sqlite.0'
-    db = Path(__file__).absolute().parent / 'data' / 'kobo_notes' / 'input' / 'KoboReader.sqlite'
+
+def get_test_db() -> Path:
+    testdata = Path(__file__).absolute().parent.parent.parent.parent / 'testdata'
+    assert testdata.exists(), testdata
+    db = testdata / 'kobo_notes' / 'input' / 'KoboReader.sqlite'
+    assert db.exists(), db
     return db
 
 # a bit meh, but ok for now
@@ -55,7 +57,7 @@ def test_books_with_highlights():
     assert ann.kind == 'annotation'
     assert ann.text == 'He does this by finding which machine has the biggest queue of materials waiting behind it and finds a way to increase its efficiency.'
     assert ann.annotation == 'Bottleneck'
-    assert ann.dt == datetime(year=2017, month=8, day=12, hour=3, minute=49, second=13, microsecond=0, tzinfo=pytz.utc)
+    assert ann.dt == datetime(year=2017, month=8, day=12, hour=3, minute=49, second=13, microsecond=0, tzinfo=timezone.utc)
     assert ann.book.author == 'Greg McKeown'
 
     assert len(pages) == 7

tests/test_rows.py → src/kobuddy/tests/test_rows.py

@@ -1,8 +1,6 @@
 from collections import OrderedDict
 from datetime import datetime
 
-import pytz
-
 import kobuddy
 from kobuddy import Highlight, Book, Books
 

tox.ini

@@ -1,35 +1,44 @@
 [tox]
-minversion = 3.5
+minversion = 3.21
 # relies on the correct version of Python installed
-envlist = tests,mypy
+envlist = ruff,tests,mypy
 # https://github.com/tox-dev/tox/issues/20#issuecomment-247788333
 # hack to prevent .tox from crapping to the project directory
-toxworkdir={env:TOXWORKDIR_BASE:}{toxinidir}/.tox
+toxworkdir = {env:TOXWORKDIR_BASE:}{toxinidir}/.tox
 
 [testenv]
 # TODO how to get package name from setuptools?
 package_name = "kobuddy"
 passenv =
 # useful for tests to know they are running under ci
-  CI
-  CI_*
+    CI
+    CI_*
 # respect user's cache dirs to prevent tox from crapping into project dir
-  MYPY_CACHE_DIR
-  PYTHONPYCACHEPREFIX
+    PYTHONPYCACHEPREFIX
+    MYPY_CACHE_DIR
+    RUFF_CACHE_DIR
 
 
+[testenv:ruff]
+commands =
+    {envpython} -m pip install --use-pep517 -e .[testing]
+    {envpython} -m ruff src/
+
+
+# note: --use-pep517 here is necessary for tox --parallel flag to work properly
+# otherwise it seems that it tries to modify .eggs dir in parallel and it fails
 [testenv:tests]
 commands =
-    {envpython} -m pip install -e .[testing]
+    {envpython} -m pip install --use-pep517 -e .[testing]
     # posargs allow test filtering, e.g. tox ... -- -k test_name
     {envpython} -m pytest \
-        tests \
+        --pyargs {[testenv]package_name} \
         {posargs}
 
 
 [testenv:mypy]
 commands =
-    {envpython} -m pip install -e .[linting]
+    {envpython} -m pip install --use-pep517 -e .[testing]
     {envpython} -m mypy --install-types --non-interactive \
         -p {[testenv]package_name}       \
         # txt report is a bit more convenient to view on CI