From 278dda702a565bdf8ec7a671373307d40b7f88f0 Mon Sep 17 00:00:00 2001
From: Kaviraj <kavirajkanagaraj@gmail.com>
Date: Thu, 21 Jan 2021 09:59:17 +0100
Subject: [PATCH 1/2] Terraform script to automate GCP provisioning for gcplog

---
 .gitignore               |  5 +++++
 scripts/gcplog/README.md | 45 ++++++++++++++++++++++++++++++++++++++++
 scripts/gcplog/main.tf   | 40 +++++++++++++++++++++++++++++++++++
 3 files changed, 90 insertions(+)
 create mode 100644 scripts/gcplog/README.md
 create mode 100644 scripts/gcplog/main.tf

diff --git a/.gitignore b/.gitignore
index d1994bf0ae09..cc5171101664 100644
--- a/.gitignore
+++ b/.gitignore
@@ -31,3 +31,8 @@ coverage.txt
 
 # emacs
 .#*
+
+# terraform
+.terraform*
+*.tfstate*
+*.tfvars
\ No newline at end of file
diff --git a/scripts/gcplog/README.md b/scripts/gcplog/README.md
new file mode 100644
index 000000000000..9351eaca4dd7
--- /dev/null
+++ b/scripts/gcplog/README.md
@@ -0,0 +1,45 @@
+# Cloud provisioning for GCP logs
+
+This document covers how to configure your GCP via Terraform to make cloud logs available for `promtail` to consume.
+
+## Prerequisite
+- Terraform >= 0.14.5
+- GCP Service account credentials with following roles/permissions
+  - "roles/pubsub.editor"
+  - "roles/logging.configWriter"
+
+## Usage
+
+```bash
+terraform init
+```
+
+```bash
+terraform plan
+```
+
+```bash
+terraform apply
+```
+
+Terraform will prompt for following variables.
+
+1. credentials_file - ServiceAccount credentials file with permissions mentioned in the prerequisite.
+2. zone - GCP zone (e.g: `us-central1-b`)
+3. region - GCP region (e.g: `us-central1`)
+4. project - GCP Project ID
+5. logname - Logname is the name we use to create pubsub topics, log router and pubsub subscription.
+
+you can pass these variables via CLI.
+
+e.g:
+```bash
+terraform apply \
+-var="credentials_file=./permissions.json" \
+-var="zone=us-central1-b" \
+-var="region=us-central1" \
+-var="project=grafanalabs-dev" \
+-var="logname=cloud-logs"
+```
+
+These variables can be passed in multiple ways. For complete reference refer terraform [doc](https://www.terraform.io/docs/configuration/variables.html#assigning-values-to-root-module-variables)
diff --git a/scripts/gcplog/main.tf b/scripts/gcplog/main.tf
new file mode 100644
index 000000000000..43d677b6747c
--- /dev/null
+++ b/scripts/gcplog/main.tf
@@ -0,0 +1,40 @@
+terraform {
+  required_providers {
+    google = {
+      source = "hashicorp/google"
+      version = "3.5.0"
+    }
+  }
+}
+
+variable "credentials_file" {}
+variable "zone" {}
+variable "region" {}
+variable "project" {}
+variable "logname" {
+  default = "cloud-logs"
+}
+
+provider "google" {
+  credentials = file(var.credentials_file)
+  project = var.project
+  zone = var.zone
+  region= var.region
+
+}
+
+resource "google_pubsub_topic" "cloud-logs" {
+  name= var.logname
+}
+
+resource "google_logging_project_sink" "cloud-logs" {
+  name = var.logname
+  destination = "pubsub.googleapis.com/projects/personal-226821/topics/${var.logname}"
+  filter = "resource.type = gcs_bucket AND severity >= WARNING"
+  unique_writer_identity = true
+}
+
+resource "google_pubsub_subscription" "coud-logs" {
+  name = var.logname
+  topic = google_pubsub_topic.cloud-logs.name
+}

From b5b8a01f05f74505c39e9b9f8e121abe2355a867 Mon Sep 17 00:00:00 2001
From: Kaviraj <kavirajkanagaraj@gmail.com>
Date: Mon, 25 Jan 2021 11:04:33 +0100
Subject: [PATCH 2/2] PR remarks

---
 scripts/gcplog/README.md | 15 +++++++++++++++
 scripts/gcplog/main.tf   |  2 +-
 2 files changed, 16 insertions(+), 1 deletion(-)

diff --git a/scripts/gcplog/README.md b/scripts/gcplog/README.md
index 9351eaca4dd7..f653a7a8f159 100644
--- a/scripts/gcplog/README.md
+++ b/scripts/gcplog/README.md
@@ -2,6 +2,21 @@
 
 This document covers how to configure your GCP via Terraform to make cloud logs available for `promtail` to consume.
 
+To choose what logs need to exported from Google Cloud, we use log filters. Log filters are normal GCP logging queries except the goal is export logs from specific set Google cloud resources
+
+e.g: Export Google APP Engine logs
+```bash
+resource.type="gae_app" AND
+severity>=ERROR
+```
+
+e.g: Export Google HTTP Loadbalancer logs
+```bash
+resource.type="http_load_balancer" AND
+httpRequest.status>=500
+```
+You can read more about these log filters in [GCP logging](https://cloud.google.com/logging/docs/view/query-library)
+
 ## Prerequisite
 - Terraform >= 0.14.5
 - GCP Service account credentials with following roles/permissions
diff --git a/scripts/gcplog/main.tf b/scripts/gcplog/main.tf
index 43d677b6747c..54a1a6c5e417 100644
--- a/scripts/gcplog/main.tf
+++ b/scripts/gcplog/main.tf
@@ -30,7 +30,7 @@ resource "google_pubsub_topic" "cloud-logs" {
 resource "google_logging_project_sink" "cloud-logs" {
   name = var.logname
   destination = "pubsub.googleapis.com/projects/personal-226821/topics/${var.logname}"
-  filter = "resource.type = gcs_bucket AND severity >= WARNING"
+  filter = "resource.type = http_load_balancer AND httpRequest.status >= 200"
   unique_writer_identity = true
 }