Bitte version

flake.nix

@@ -41,6 +41,8 @@
 
     nix.url = "github:NixOS/nix/c6fa7775de413a799b9a137dceced5dcf0f5e6ed";
     nix.inputs.nixpkgs.follows = "nixpkgs";
+
+    nixpkgs-ext4-ami.url = "github:NixOS/nixpkgs/585bc245438331f19d8621646fb8d69a95b42815";
   };
 
   outputs =
@@ -110,6 +112,7 @@
         , ci-env
         , mkRequired
         , asgAMI
+        , zfsAsgAMI
         }@pkgs:
         let constituents = builtins.removeAttrs pkgs [ "mkRequired" ];
         in constituents // { required = mkRequired constituents; };

lib/clusters.nix

@@ -1,11 +1,22 @@
-{ self, pkgs, system, lib, root, ... }:
+{ self, pkgs, system, lib, root
+, bitteVersion ? 1
+, ...
+}:
 
 let
   inherit (builtins) attrNames readDir mapAttrs;
   inherit (lib)
     flip pipe mkForce filterAttrs flatten listToAttrs forEach nameValuePair
     mapAttrs';
 
+  amazonImageModule = if bitteVersion == 1
+    then
+      (self.inputs.nixpkgs-ext4 + "/nixos/maintainers/scripts/ec2/amazon-image.nix");
+    else
+      (self.inputs.nixpkgs + "/nixos/maintainers/scripts/ec2/amazon-image-zfs.nix")
+
+
+
   readDirRec = path:
     pipe path [
       readDir
@@ -25,24 +36,9 @@ let
       inherit pkgs system;
       modules = [
         self.inputs.bitte.nixosModule
-        (self.inputs.nixpkgs + "/nixos/modules/virtualisation/amazon-image.nix")
-      ] ++ modules;
-      specialArgs = { inherit nodeName self; };
-    };
-
-  mkAMI = nodeName: modules:
-    self.inputs.nixpkgs.lib.nixosSystem {
-      inherit pkgs system;
-      modules = [
-        self.inputs.bitte.nixosModule
-        ({ modulesPath, ... }: {
-          imports = [
-            "${modulesPath}/../maintainers/scripts/ec2/amazon-image-zfs.nix"
-          ];
-          services.openssh.enable = true;
-        })
+        amazonImageModule
       ] ++ modules;
-      specialArgs = { inherit nodeName self; };
+      specialArgs = { inherit nodeName self bitteVersion; };
     };
 
   clusterFiles = readDirRec root;
@@ -67,9 +63,6 @@ in listToAttrs (forEach clusterFiles (file:
       ([ { networking.hostName = mkForce name; } file ] ++ instance.modules))
       proto.config.cluster.instances;
 
-    ami = mapAttrs (name: instance: mkAMI name ([ file ] ++ instance.modules))
-      proto.config.cluster.autoscalingGroups;
-
     groups =
       mapAttrs (name: instance: mkSystem name ([ file ] ++ instance.modules))
       proto.config.cluster.autoscalingGroups;

modules/terraform.nix

@@ -1,4 +1,4 @@
-{ self, config, pkgs, nodeName, ... }:
+{ self, config, pkgs, nodeName, bitteVersion, ... }:
 let
   inherit (pkgs) lib terralib;
   inherit (lib) mkOption reverseList;
@@ -19,12 +19,14 @@ let
       nixosAmis.latest;
   };
 
-  autoscalingAMIs = {
+  autoscalingAMIs = if bitteVersion == 1 then {
     eu-central-1 = "ami-07cf06fc2cf0de485";
     us-east-2 = "ami-08c2048194fde1422";
     eu-west-1 = "ami-0ac83c4afcc9e6ecc";
     us-east-1 = "ami-0baa6fb5107677998";
-  };
+  } else
+  # TODO: 21.11 nixos amis should be compatible
+  lib.throw "Autoscaling ami's must be generated and upload per cluster";
 
   vpcMap = lib.pipe [
     "ap-northeast-1"
@@ -683,7 +685,29 @@ let
         '';
       in mkOption {
         type = nullOr str;
-        default = ''
+        default = if bitteVersion == 1 then ''
+          # amazon-shell-init
+          set -exuo pipefail
+
+          /run/current-system/sw/bin/zpool online -e tank nvme0n1p3
+
+          export CACHES="https://hydra.iohk.io https://cache.nixos.org ${cfg.s3Cache}"
+          export CACHE_KEYS="hydra.iohk.io:f/Ea+s+dFdN+3Y/G+FDgSq+a5NEWhJGzdjvKNGv0/EQ= cache.nixos.org-1:6NCHdD59X431o0gWypbMrAURkbJ16ZPMQFGspcDShjY= ${cfg.s3CachePubKey}"
+
+          pushd /run/keys
+          aws s3 cp "s3://${cfg.s3Bucket}/infra/secrets/${cfg.name}/${cfg.kms}/source/source.tar.xz" source.tar.xz
+          mkdir -p source
+          tar xvf source.tar.xz -C source
+
+          # TODO: add git to the AMI
+          nix build nixpkgs#git -o git
+          export PATH="$PATH:$PWD/git/bin"
+
+          nix build ./source#nixosConfigurations.${cfg.name}-${this.config.name}.config.system.build.toplevel --option substituters "$CACHES" --option trusted-public-keys "$CACHE_KEYS"
+          /run/current-system/sw/bin/nixos-rebuild --flake ./source#${cfg.name}-${this.config.name} boot --option substituters "$CACHES" --option trusted-public-keys "$CACHE_KEYS"
+
+          /run/current-system/sw/bin/shutdown -r now
+        '' else ''
           #!/usr/bin/env bash
           export NIX_CONFIG="${nixConf}"
 

overlay.nix

@@ -309,12 +309,27 @@ in final: prev: {
     inherit (keys) allKeysFrom devOps;
   in { devOps = allKeysFrom devOps; };
 
-  mkAsgAMI = import ./pkgs/ami.nix;
+  mkAsgAMI = { nixpkgs, system, extraModules ? [ ] }@args:
+  (import ./pkgs/ami.nix) (args // {
+    extraModules = [ (inputs.nixpkgs-ext4-ami + "/nixos/maintainers/scripts/ec2/amazon-image.nix") ]
+      ++ args.extraModules;
+  });
+
+  mkZfsAsgAMI = { nixpkgs, system, extraModules ? [ ] }@args:
+  (import ./pkgs/ami.nix) (args // {
+    extraModules = [ (inputs.nixpkgs + "/nixos/maintainers/scripts/ec2/amazon-image-zfs.nix") ]
+      ++ args.extraModules;
+  });
 
   asgAMI = (final.mkAsgAMI {
     inherit nixpkgs;
     inherit (prev) system;
     extraModules = [ ];
   }).config.system.build.amazonImage;
 
+  zfsAsgAMI = (final.mkZfsAsgAMI {
+    inherit nixpkgs;
+    inherit (prev) system;
+    extraModules = [ ];
+  }).config.system.build.amazonImage;
 }

pkgs/ami.nix

@@ -2,8 +2,6 @@
 let
   modules = [
     ({ pkgs, modulesPath, ... }: {
-      imports =
-        [ "${modulesPath}/../maintainers/scripts/ec2/amazon-image-zfs.nix" ];
       nix.package = pkgs.nixUnstable;
       nix.binaryCaches = [ "https://hydra.iohk.io" ];
       nix.binaryCachePublicKeys =