From 90c2ef78abe757f994299087ab18bb1a7d78481f Mon Sep 17 00:00:00 2001 From: Robert Veznaver Date: Mon, 12 Dec 2016 17:36:36 +0100 Subject: [PATCH 1/4] Fix bug in sparse key naming Signed-off-by: Robert Veznaver --- lib/chef-vault/item_keys.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/chef-vault/item_keys.rb b/lib/chef-vault/item_keys.rb index 6927255..c49193b 100644 --- a/lib/chef-vault/item_keys.rb +++ b/lib/chef-vault/item_keys.rb @@ -205,7 +205,7 @@ def self.load(vault, name) # @private def sparse_id(key, item_id = @raw_data["id"]) - "#{item_id}_key_#{key}" + "#{item_id.chomp("_keys")}_key_#{key}" end def sparse_key(sid) From d9d9700ca0b2696382285b95908c49ad859ff253 Mon Sep 17 00:00:00 2001 From: Robert Veznaver Date: Fri, 9 Dec 2016 16:30:35 +0100 Subject: [PATCH 2/4] Add feature to create sparse keys with knife Signed-off-by: Robert Veznaver --- features/step_definitions/chef-vault.rb | 27 +++++++++++++++++-------- features/vault_create.feature | 10 +++++++++ lib/chef/knife/vault_create.rb | 7 +++++++ 3 files changed, 36 insertions(+), 8 deletions(-) diff --git a/features/step_definitions/chef-vault.rb b/features/step_definitions/chef-vault.rb index 37cefd5..d87a333 100644 --- a/features/step_definitions/chef-vault.rb +++ b/features/step_definitions/chef-vault.rb @@ -1,10 +1,11 @@ require "json" -Given(/^I create a vault item '(.+)\/(.+)' containing the JSON '(.+)' encrypted for '(.+)'(?: with '(.+)' as admins?)?$/) do |vault, item, json, nodelist, admins| +Given(/^I create a vault item '(.+)\/(.+)'( with keys in sparse mode)? containing the JSON '(.+)' encrypted for '(.+)'(?: with '(.+)' as admins?)?$/) do |vault, item, sparse, json, nodelist, admins| write_file "item.json", json query = nodelist.split(/,/).map { |e| "name:#{e}" }.join(" OR ") adminarg = admins.nil? ? "-A admin" : "-A #{admins}" - run_simple "knife vault create #{vault} #{item} -z -c knife.rb #{adminarg} -S '#{query}' -J item.json", false + sparseopt = sparse.nil? ? '' : '-K sparse' + run_simple "knife vault create #{vault} #{item} -z -c knife.rb #{adminarg} #{sparseopt} -S '#{query}' -J item.json", false end Given(/^I update the vault item '(.+)\/(.+)' to be encrypted for '(.+)'( with the clean option)?$/) do |vault, item, nodelist, cleanopt| @@ -41,18 +42,28 @@ run_simple "knife vault show #{vault} #{item} -z -c knife.rb -u #{node} -k #{node}.pem", false end -Then(/^the vault item '(.+)\/(.+)' should( not)? be encrypted for '(.+)'$/) do |vault, item, neg, nodelist| +Then(/^the vault item '(.+)\/(.+)' should( not)? be encrypted for '(.+)'( with keys in sparse mode)?$/) do |vault, item, neg, nodelist, sparse| nodes = nodelist.split(/,/) command = "knife data bag show #{vault} #{item}_keys -z -c knife.rb -F json" run_simple(command) output = last_command_started.stdout data = JSON.parse(output) - nodes.each do |node| - if neg - expect(data).not_to include(node) - else - expect(data).to include(node) + if sparse + expect(data).to include('mode' => 'sparse') + nodes.each do |node| + command = "knife data bag show #{vault} #{item}_key_#{node} -z -c knife.rb -F json" + run_simple(command, fail_on_error: false) + if neg + error = last_command_started.stderr + expect(error).to include('ERROR: The object you are looking for could not be found') + else + data = JSON.parse(last_command_started.stdout) + expect(data).to include('id' => "#{item}_key_#{node}") + end end + else + expect(data).to include('mode' => 'default') + nodes.each { |node| neg ? (expect(data).not_to include(node)) : (expect(data).to include(node)) } end end diff --git a/features/vault_create.feature b/features/vault_create.feature index 8f64d66..2c78513 100644 --- a/features/vault_create.feature +++ b/features/vault_create.feature @@ -46,6 +46,16 @@ Feature: knife vault create And 'alice' should be an admin for the vault item 'test/item' And 'bob' should not be an admin for the vault item 'test/item' + Scenario: create vault with several admins in sparse mode + Given a local mode chef repo with nodes 'one,two' with admins 'alice,bob' + And I create a vault item 'test/item' with keys in sparse mode containing the JSON '{"foo": "bar"}' encrypted for 'one,two,three' with 'alice' as admin + Then the vault item 'test/item' should be encrypted for 'one,two' with keys in sparse mode + And the vault item 'test/item' should not be encrypted for 'three' with keys in sparse mode + And 'one,two' should be a client for the vault item 'test/item' + And 'three' should not be a client for the vault item 'test/item' + And 'alice' should be an admin for the vault item 'test/item' + And 'bob' should not be an admin for the vault item 'test/item' + Scenario: create vault with an unknown admin Given a local mode chef repo with nodes 'one,two' And I create a vault item 'test/item' containing the JSON '{"foo": "bar"}' encrypted for 'one,two,three' with 'alice' as admin diff --git a/lib/chef/knife/vault_create.rb b/lib/chef/knife/vault_create.rb index ac4688e..536ea70 100644 --- a/lib/chef/knife/vault_create.rb +++ b/lib/chef/knife/vault_create.rb @@ -26,6 +26,11 @@ class VaultCreate < Knife banner "knife vault create VAULT ITEM VALUES (options)" + option :keys_mode, + :short => "-K KEYS_MODE", + :long => "--keys-mode KEYS_MODE", + :description => "Mode in which to save vault keys" + option :search, :short => "-S SEARCH", :long => "--search SEARCH", @@ -57,6 +62,7 @@ def run search = config[:search] json_file = config[:json] file = config[:file] + keys_mode = config[:keys_mode] set_mode(config[:vault_mode]) @@ -91,6 +97,7 @@ def run vault_item.clients if search vault_item.clients(clients) if clients vault_item.admins(admins) if admins + vault_item.keys.mode(keys_mode) if keys_mode vault_item.save end From 27ac0e8d75a613a6e4277d94f2d83f7772d1b42a Mon Sep 17 00:00:00 2001 From: Robert Veznaver Date: Mon, 19 Dec 2016 17:00:07 +0100 Subject: [PATCH 3/4] Cleanup all sparse keys on destroy Signed-off-by: Robert Veznaver --- features/step_definitions/chef-vault.rb | 10 ++--- lib/chef-vault/item_keys.rb | 17 ++++++-- spec/chef-vault/item_keys_spec.rb | 58 ++++++++++++++++++++++--- 3 files changed, 71 insertions(+), 14 deletions(-) diff --git a/features/step_definitions/chef-vault.rb b/features/step_definitions/chef-vault.rb index d87a333..fec5a35 100644 --- a/features/step_definitions/chef-vault.rb +++ b/features/step_definitions/chef-vault.rb @@ -4,7 +4,7 @@ write_file "item.json", json query = nodelist.split(/,/).map { |e| "name:#{e}" }.join(" OR ") adminarg = admins.nil? ? "-A admin" : "-A #{admins}" - sparseopt = sparse.nil? ? '' : '-K sparse' + sparseopt = sparse.nil? ? "" : "-K sparse" run_simple "knife vault create #{vault} #{item} -z -c knife.rb #{adminarg} #{sparseopt} -S '#{query}' -J item.json", false end @@ -49,20 +49,20 @@ output = last_command_started.stdout data = JSON.parse(output) if sparse - expect(data).to include('mode' => 'sparse') + expect(data).to include("mode" => "sparse") nodes.each do |node| command = "knife data bag show #{vault} #{item}_key_#{node} -z -c knife.rb -F json" run_simple(command, fail_on_error: false) if neg error = last_command_started.stderr - expect(error).to include('ERROR: The object you are looking for could not be found') + expect(error).to include("ERROR: The object you are looking for could not be found") else data = JSON.parse(last_command_started.stdout) - expect(data).to include('id' => "#{item}_key_#{node}") + expect(data).to include("id" => "#{item}_key_#{node}") end end else - expect(data).to include('mode' => 'default') + expect(data).to include("mode" => "default") nodes.each { |node| neg ? (expect(data).not_to include(node)) : (expect(data).to include(node)) } end end diff --git a/lib/chef-vault/item_keys.rb b/lib/chef-vault/item_keys.rb index c49193b..91b16a6 100644 --- a/lib/chef-vault/item_keys.rb +++ b/lib/chef-vault/item_keys.rb @@ -161,14 +161,23 @@ def save(item_id = @raw_data["id"]) def destroy if Chef::Config[:solo_legacy_mode] - data_bag_path = File.join(Chef::Config[:data_bag_path], - data_bag) + data_bag_path = File.join(Chef::Config[:data_bag_path], data_bag) data_bag_item_path = File.join(data_bag_path, @raw_data["id"]) - + data_bag_sparse_keys_path = File.join(data_bag_path, sparse_id("*")) + # destroy all sparse keys + FileUtils.rm(Dir.glob("#{data_bag_sparse_keys_path}.json")) + # destroy this metadata FileUtils.rm("#{data_bag_item_path}.json") - nil else + # destroy all sparse keys + rgx = Regexp.new("^#{sparse_id(".*")}") + items = Chef::DataBag.load(data_bag).keys.select { |item| item =~ rgx } + items.each do |id| + Chef::DataBagItem.from_hash("data_bag" => data_bag, "id" => id) + .destroy(data_bag, id) + end + # destroy this metadata super(data_bag, id) end end diff --git a/spec/chef-vault/item_keys_spec.rb b/spec/chef-vault/item_keys_spec.rb index 9820064..e464132 100644 --- a/spec/chef-vault/item_keys_spec.rb +++ b/spec/chef-vault/item_keys_spec.rb @@ -81,11 +81,8 @@ describe "#save" do let(:client_name) { "client_name" } - let(:chef_key) { ChefVault::Actor.new("clients", client_name) } - - before do - allow(chef_key).to receive(:key) { public_key_string } - end + let(:chef_key) { ChefVault::Actor.new("clients", client_name) } + before { allow(chef_key).to receive(:key) { public_key_string } } it "should save the key data" do keys.add(chef_key, shared_secret) @@ -110,6 +107,29 @@ keys.mode("default") end end + + describe "#destroy" do + let(:client_name) { "client_name" } + let(:chef_key) { ChefVault::Actor.new("clients", client_name) } + before { allow(chef_key).to receive(:key) { public_key_string } } + + it "should destroy the keys" do + keys.add(chef_key, shared_secret) + keys.save("bar") + keys.destroy + expect { Chef::DataBagItem.load("foo", "bar") }.to raise_error(Net::HTTPServerException) + end + + it "should destroy the keys in sparse mode" do + keys.add(chef_key, shared_secret) + keys.mode("sparse") + keys.save("bar") + keys.destroy + expect { Chef::DataBagItem.load("foo", "bar") }.to raise_error(Net::HTTPServerException) + expect { Chef::DataBagItem.load("foo", "bar_key_client_name") }.to raise_error(Net::HTTPServerException) + keys.mode("default") + end + end end context "when running with chef-solo" do @@ -182,6 +202,34 @@ keys.mode("default") end end + + describe "#destroy" do + let(:client_name) { "client_name" } + let(:chef_key) { ChefVault::Actor.new("clients", client_name) } + let(:data_bag_path) { Dir.mktmpdir("vault_item_keys") } + + before do + Chef::Config[:data_bag_path] = data_bag_path + allow(chef_key).to receive(:key) { public_key_string } + end + + it "should destroy the keys" do + keys.add(chef_key, shared_secret) + keys.save("bar") + keys.destroy + expect(File.exist?(File.join(data_bag_path, "foo", "bar.json"))).to be(false) + end + + it "should destroy the keys in sparse mode" do + keys.add(chef_key, shared_secret) + keys.mode("sparse") + keys.save("bar") + keys.destroy + expect(File.exist?(File.join(data_bag_path, "foo", "bar.json"))).to be(false) + expect(File.exist?(File.join(data_bag_path, "foo", "bar_key_client_name.json"))).to be(false) + keys.mode("default") + end + end end end end From d17caf9baeaa1a6b78e507e054b0555cbc365cac Mon Sep 17 00:00:00 2001 From: Robert Veznaver Date: Tue, 20 Dec 2016 16:40:46 +0100 Subject: [PATCH 4/4] Extend vault detection to sparse mode Signed-off-by: Robert Veznaver --- features/isvault.feature | 6 ++++++ features/vault_list.feature | 6 ++++++ lib/chef/knife/vault_base.rb | 19 +++++++++++++++---- 3 files changed, 27 insertions(+), 4 deletions(-) diff --git a/features/isvault.feature b/features/isvault.feature index a7bf4fc..9010777 100644 --- a/features/isvault.feature +++ b/features/isvault.feature @@ -8,6 +8,12 @@ Feature: determine if a data bag item is a vault And I check if the data bag item 'test/item' is a vault Then the exit status should be 0 + Scenario: detect vault item with keys in sparse mode + Given a local mode chef repo with nodes 'one,two,three' + And I create a vault item 'test/item' with keys in sparse mode containing the JSON '{"foo": "bar"}' encrypted for 'one,two,three' + And I check if the data bag item 'test/item' is a vault + Then the exit status should be 0 + Scenario: detect non-vault item (encrypted data bag) Given a local mode chef repo with nodes 'one,two,three' And I create an empty data bag 'test' diff --git a/features/vault_list.feature b/features/vault_list.feature index a6abcb0..3396019 100644 --- a/features/vault_list.feature +++ b/features/vault_list.feature @@ -10,6 +10,12 @@ Feature: list data bags that are vaults And I list the vaults Then the output should match /(?m:^test$)/ + Scenario: List bags that are vaults with keys in sparse mode + Given a local mode chef repo with nodes 'one,two,three' + And I create a vault item 'test/item' with keys in sparse mode containing the JSON '{"foo": "bar"}' encrypted for 'one,two,three' + And I list the vaults + Then the output should match /(?m:^test$)/ + Scenario: Skip data bags that are not vaults Given a local mode chef repo with nodes 'one,two,three' And I create a vault item 'test/item' containing the JSON '{"foo": "bar"}' encrypted for 'one,two,three' diff --git a/lib/chef/knife/vault_base.rb b/lib/chef/knife/vault_base.rb index 8b7588c..5ca8f7c 100644 --- a/lib/chef/knife/vault_base.rb +++ b/lib/chef/knife/vault_base.rb @@ -49,8 +49,12 @@ def configure_chef def bag_is_vault?(bagname) bag = Chef::DataBag.load(bagname) - # vaults have at even number of keys >= 2 - return false unless bag.keys.size >= 2 && 0 == bag.keys.size % 2 + # a data bag is a vault if and only if: + # - it has at least one item with item_keys + # - every item has a matching item_keys + # - item_keys has zero or more keys in sparse mode + # vaults have a number of keys >= 2 + return false unless bag.keys.size >= 2 # partition into those that end in _keys keylike, notkeylike = split_vault_keys(bag) # there must be an equal number of keyline and not-keylike items @@ -63,8 +67,15 @@ def bag_is_vault?(bagname) end def split_vault_keys(bag) - # partition into those that end in _keys - bag.keys.partition { |k| k =~ /_keys$/ } + # get all item keys + keys = bag.keys.select { |k| k =~ /_keys$/ } + # get all sparse keys + r = Regexp.union(keys.map { |k| Regexp.new("^#{k.chomp('_keys')}_key_.*") }) + sparse = bag.keys.select { |k| k =~ r } + # the rest + items = bag.keys - keys - sparse + # return item keys and items + [keys, items] end end end