Update build.sh

abuseipdb + version enhancements

ci debug

.devcontainer/devcontainer.json

@@ -3,5 +3,5 @@
     "features": {
         "ghcr.io/devcontainers/features/python:1": {"version": "3.11"}
     },
-    "onCreateCommand": "bash -i install.sh"
+    "onCreateCommand": "python install.py"
 }

.github/workflows/deploy.yaml

@@ -1,4 +1,4 @@
-name: Deploy to GitHub Pages and build js release
+name: Deploy to GitHub Pages and build release if tagged
 
 permissions:
   contents: write
@@ -13,10 +13,10 @@ jobs:
       uses: fastai/workflows/quarto-ghp@master
       with:
         version: '3.11' 
-    - name: npm build
-      run: npm install && npm run build
-    - name: npm release upload
+    - name: build
+      run: ./build.sh
+    - name: release upload
       uses: softprops/action-gh-release@v1
       if: startsWith(github.ref, 'refs/tags/')
       with:
-        files: atlaskit-transformer.bundle.js
+        files: dist/*

MANIFEST.in

@@ -2,5 +2,5 @@ include settings.ini
 include LICENSE
 include CONTRIBUTING.md
 include README.md
-include squ/atlaskit-transformer.bundle.js
+include nbdev_squ/atlaskit-transformer.bundle.js
 recursive-exclude * __pycache__

README.md

@@ -8,7 +8,7 @@
 Below is how to install in a plain python 3.11+ environment
 
 ``` sh
-pip install https://github.com/wagov/nbdev-squ/archive/refs/tags/v1.3.1.tar.gz
+https://github.com/wagov/nbdev-squ/releases/download/v1.3.2/nbdev_squ-1.3.2-py3-none-any.whl
 ```
 
 The installation can also be run in a notebook (we tend to use
@@ -18,7 +18,7 @@ should load the json secret *squconfig-`my_keyvault_tenantid`* from the
 `my_kevault_name` keyvault.
 
 ``` python
-%pip install https://github.com/wagov/nbdev-squ/archive/refs/tags/v1.3.1.tar.gz
+%pip install https://github.com/wagov/nbdev-squ/releases/download/v1.3.2/nbdev_squ-1.3.2-py3-none-any.whl
 import os; os.environ["SQU_CONFIG"] = "{{ my_keyvault_name }}/{{ my_keyvault_tenantid }}" 
 
 from nbdev_squ import api

build.sh

@@ -0,0 +1,6 @@
+#!/bin/bash
+# Just used during CI to build static assets and add to release
+npm install
+npm run build
+pip install build
+python -m build

install.py

@@ -0,0 +1,26 @@
+#!/usr/bin/env python
+# This is meant to be run to setup and prep for committing a release
+# use nbdev_bump_version to increment the version itself then rerun this to update README.md _docs etc
+from subprocess import run
+import configparser, re
+
+run(["pip", "install", "nbdev"])
+run(["pip", "install", "-e", "."]) # get current project in dev mode
+run("quarto --version || nbdev_install", shell=True)
+run(["npx", "npm-check-updates", "-u"]) # convenient way to freshen package.json on each release
+run(["npm", "install"])
+run(["npm", "run", "build"])
+run(["nbdev_clean"])
+run(["nbdev_export"])
+
+config = configparser.ConfigParser()
+config.read('settings.ini')
+version = config.get("DEFAULT", "version")
+git_url = config.get("DEFAULT", "git_url")
+latest_download = f"{git_url}/releases/download/v{version}/nbdev_squ-{version}-py3-none-any.whl"
+index_text = open("nbs/index.ipynb").read()
+with open("nbs/index.ipynb", "w") as index_nb:
+    index_nb.write(re.sub(f"{git_url}.*?-any.whl", latest_download, index_text))
+
+run(["nbdev_readme"])
+run(["nbdev_docs"])

nbdev_squ/__init__.py

@@ -1 +1 @@
-__version__ = "1.3.1"
+__version__ = "1.3.2"

nbdev_squ/api.py

@@ -6,17 +6,15 @@
            'atlaskit_transformer', 'security_incidents', 'security_alerts']
 
 # %% ../nbs/01_api.ipynb 3
-import pandas, json, logging, time, requests, httpx_cache, io
+import pandas, json, logging, time, requests, io, pkgutil
 from .core import *
 from diskcache import memoize_stampede
+from importlib.metadata import version
 from subprocess import run, CalledProcessError
 from azure.monitor.query import LogsQueryClient, LogsBatchQuery, LogsQueryStatus
 from azure.identity import AzureCliCredential
 from benedict import benedict
 from functools import cached_property
-from abuseipdb_wrapper import AbuseIPDB
-from atlassian import Jira
-from tenable.io import TenableIO
 
 # %% ../nbs/01_api.ipynb 5
 logger = logging.getLogger(__name__)
@@ -36,27 +34,31 @@ def runzero(self):
         """
         Returns a runzero client
         """
+        import httpx_cache
         return httpx_cache.Client(base_url="https://console.rumble.run/api/v1.0", headers={"Authorization": f"Bearer {self.config.runzero_apitoken}"})
 
     @cached_property
     def abuseipdb(self):
         """
         Returns an abuseipdb client
         """
-        return AbuseIPDB(API_KEY=self.config.abuseipdb_api_key)
+        from abuseipdb_wrapper import AbuseIPDB
+        return AbuseIPDB(api_key=self.config.abuseipdb_api_key)
 
     @cached_property
     def jira(self):
         """
         Returns a jira client
         """
+        from atlassian import Jira
         return Jira(url=self.config.jira_url, username=self.config.jira_username, password=self.config.jira_password)
 
     @cached_property
     def tio(self):
         """
         Returns a TenableIO client
         """
+        from tenable.io import TenableIO
         return TenableIO(self.config.tenable_access_key, self.config.tenable_secret_key)
 
 
@@ -237,11 +239,9 @@ def hunt(indicators, expression="has", columns=columns, workspaces=None, timespa
 
 # %% ../nbs/01_api.ipynb 22
 def atlaskit_transformer(inputtext, inputfmt="md", outputfmt="wiki", runtime="node"):
-    import nbdev_squ
-    transformer = dirs.user_cache_path / f"atlaskit-transformer.bundle_v{nbdev_squ.__version__}.js"
+    transformer = dirs.user_cache_path / f"atlaskit-transformer.bundle_v{version('nbdev_squ')}.js"
     if not transformer.exists():
-        transformer_url = f'https://github.com/wagov/nbdev-squ/releases/download/v{nbdev_squ.__version__}/atlaskit-transformer.bundle.js'
-        transformer.write_bytes(requests.get(transformer_url).content)
+        transformer.write_bytes(pkgutil.get_data("nbdev_squ", "atlaskit-transformer.bundle.js"))
     cmd = [runtime, str(transformer), inputfmt, outputfmt]
     logger.debug(" ".join(cmd))
     try:

nbs/01_api.ipynb

@@ -35,17 +35,15 @@
    "outputs": [],
    "source": [
     "#| export\n",
-    "import pandas, json, logging, time, requests, httpx_cache, io\n",
+    "import pandas, json, logging, time, requests, io, pkgutil\n",
     "from nbdev_squ.core import *\n",
     "from diskcache import memoize_stampede\n",
+    "from importlib.metadata import version\n",
     "from subprocess import run, CalledProcessError\n",
     "from azure.monitor.query import LogsQueryClient, LogsBatchQuery, LogsQueryStatus\n",
     "from azure.identity import AzureCliCredential\n",
     "from benedict import benedict\n",
-    "from functools import cached_property\n",
-    "from abuseipdb_wrapper import AbuseIPDB\n",
-    "from atlassian import Jira\n",
-    "from tenable.io import TenableIO"
+    "from functools import cached_property"
    ]
   },
   {
@@ -97,27 +95,31 @@
     "        \"\"\"\n",
     "        Returns a runzero client\n",
     "        \"\"\"\n",
+    "        import httpx_cache\n",
     "        return httpx_cache.Client(base_url=\"https://console.rumble.run/api/v1.0\", headers={\"Authorization\": f\"Bearer {self.config.runzero_apitoken}\"})\n",
     "\n",
     "    @cached_property\n",
     "    def abuseipdb(self):\n",
     "        \"\"\"\n",
     "        Returns an abuseipdb client\n",
     "        \"\"\"\n",
-    "        return AbuseIPDB(API_KEY=self.config.abuseipdb_api_key)\n",
+    "        from abuseipdb_wrapper import AbuseIPDB\n",
+    "        return AbuseIPDB(api_key=self.config.abuseipdb_api_key)\n",
     "\n",
     "    @cached_property\n",
     "    def jira(self):\n",
     "        \"\"\"\n",
     "        Returns a jira client\n",
     "        \"\"\"\n",
+    "        from atlassian import Jira\n",
     "        return Jira(url=self.config.jira_url, username=self.config.jira_username, password=self.config.jira_password)\n",
     "\n",
     "    @cached_property\n",
     "    def tio(self):\n",
     "        \"\"\"\n",
     "        Returns a TenableIO client\n",
     "        \"\"\"\n",
+    "        from tenable.io import TenableIO\n",
     "        return TenableIO(self.config.tenable_access_key, self.config.tenable_secret_key)\n",
     "\n",
     "\n",
@@ -428,11 +430,9 @@
    "source": [
     "#| exports\n",
     "def atlaskit_transformer(inputtext, inputfmt=\"md\", outputfmt=\"wiki\", runtime=\"node\"):\n",
-    "    import nbdev_squ\n",
-    "    transformer = dirs.user_cache_path / f\"atlaskit-transformer.bundle_v{nbdev_squ.__version__}.js\"\n",
+    "    transformer = dirs.user_cache_path / f\"atlaskit-transformer.bundle_v{version('nbdev_squ')}.js\"\n",
     "    if not transformer.exists():\n",
-    "        transformer_url = f'https://github.com/wagov/nbdev-squ/releases/download/v{nbdev_squ.__version__}/atlaskit-transformer.bundle.js'\n",
-    "        transformer.write_bytes(requests.get(transformer_url).content)\n",
+    "        transformer.write_bytes(pkgutil.get_data(\"nbdev_squ\", \"atlaskit-transformer.bundle.js\"))\n",
     "    cmd = [runtime, str(transformer), inputfmt, outputfmt]\n",
     "    logger.debug(\" \".join(cmd))\n",
     "    try:\n",

nbs/index.ipynb

@@ -28,13 +28,13 @@
     "Below is how to install in a plain python 3.11+ environment\n",
     "\n",
     "```sh\n",
-    "pip install https://github.com/wagov/nbdev-squ/archive/refs/tags/v1.3.1.tar.gz\n",
+    "https://github.com/wagov/nbdev-squ/releases/download/v1.3.2/nbdev_squ-1.3.2-py3-none-any.whl\n",
     "```\n",
     "\n",
     "The installation can also be run in a notebook (we tend to use [JupyterLab Desktop](https://github.com/jupyterlab/jupyterlab-desktop) for local dev). The `SQU_CONFIG` env var indicates to nbdev_squ it should load the json secret *squconfig-`my_keyvault_tenantid`* from the `my_kevault_name` keyvault.\n",
     "\n",
     "```python\n",
-    "%pip install https://github.com/wagov/nbdev-squ/archive/refs/tags/v1.3.1.tar.gz\n",
+    "%pip install https://github.com/wagov/nbdev-squ/releases/download/v1.3.2/nbdev_squ-1.3.2-py3-none-any.whl\n",
     "import os; os.environ[\"SQU_CONFIG\"] = \"{{ my_keyvault_name }}/{{ my_keyvault_tenantid }}\" \n",
     "\n",
     "from nbdev_squ import api\n",

nbs/sidebar.yml

@@ -3,4 +3,4 @@ website:
     contents:
       - index.ipynb
       - 00_core.ipynb
-      - 01_api.ipynb
+      - 01_api.ipynb