diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..4e0bbdf --- /dev/null +++ b/.gitignore @@ -0,0 +1,2 @@ +venv/ +.idea/ diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index fb31673..2f5665f 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -1,13 +1,27 @@ -## Contributing - 1. Fork the project. - 1. Make your bug fix or new feature. - 1. Add tests for your code. - 1. Send a pull request. +# +# Copyright (c) 2021, 2022 by Delphix. All rights reserved. +# -Contributions must be signed as `User Name `. Make sure to [set up Git with user name and email address](https://git-scm.com/book/en/v2/Getting-Started-First-Time-Git-Setup). Bug fixes should branch from the current stable branch. New features should be based on the release branch. +This project is currently not accepting external contributions. -## Contributor Agreement -All contributors are required to sign the Delphix Contributor agreement prior to contributing code to an open source repository. This process is handled automatically by [cla-assistant](https://cla-assistant.io/). Simply open a pull request and a bot will automatically check to see if you have signed the latest agreement. If not, you will be prompted to do so as part of the pull request process. +
+--- Hidden Block ---- + + ## Contributing + 1. Fork the project. + 2. Refer to [README.md](README.md) for the setup, pre-configurations and running the project. + 3. Once you have run the script successfully using `make run` (Linux/MacOS) or `python src\main.py` (Windows) + 7. Make your bug fix or new feature. + 8. Add tests for your code. + 9. Send a pull request. + + Contributions must be signed as `User Name `. Make sure to [set up Git with user name and email address](https://git-scm.com/book/en/v2/Getting-Started-First-Time-Git-Setup). Bug fixes should branch from the current stable branch. New features should be based on the release branch. + + ## Contributor Agreement + All contributors are required to sign the Delphix Contributor agreement prior to contributing code to an open source repository. This process is handled automatically by [cla-assistant](https://cla-assistant.io/). Simply open a pull request and a bot will automatically check to see if you have signed the latest agreement. If not, you will be prompted to do so as part of the pull request process. + +------------------------ +
## Code of Conduct This project operates under the Delphix Code of Conduct. By participating in this project you agree to abide by its terms. diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..e3c8b82 --- /dev/null +++ b/Makefile @@ -0,0 +1,95 @@ +# +# Copyright (c) 2022 by Delphix. All rights reserved. +# + +.PHONY: --check_python --check_env_exists run tests --create_virtual_env --install_dependencies env clean_build clean_env clean build + +ROOT_DIR := $(shell pwd) +VENV := venv +PYTHON := $(VENV)/bin/python +PIP := $(VENV)/bin/pip +DLPX_NR := $(ROOT_DIR)/dist/delphix-nr +DEFAULT_PYTHON3_ALIAS := python3 + +define show_version + @export LC_ALL=en_AU.UTF-8; $(DLPX_NR) --version +endef + +define check_python_exists + @if ! command -v $(DEFAULT_PYTHON3_ALIAS) -V >/dev/null 2>&1; then \ + echo "Python 3 is NOT present on the system, Please install it"; \ + echo "If it is installed, then change the "DEFAULT_PYTHON3_ALIAS" in Makefile";\ + exit 1; \ + fi +endef + + +--check_python: + @# Help: Checks existence of python virtual environment + $(call check_python_exists) + +--check_env_exists: --check_python + @[ -f $(PYTHON) ] && echo $(PYTHON) exists || (echo $(PYTHON) does NOT exist, use \"make env\" to create a virtual env; exit 1) + + +run: --check_env_exists + @# Help: Takes care of checking the prequisites like python, virtual env, dependencies and at last shows the Python Version + -@echo `$(PYTHON) -V` + export PYTHONPATH=$(ROOT_DIR); $(PYTHON) src/main.py + +tests: + @# Help: Runs the unit tests inside tests folder and create a report + $(PYTHON) -m pytest tests -s -v + +--create_virtual_env: --check_python + @# Help: Creates a virtual environment + -@echo 'Creating Virtual environment' + @$(DEFAULT_PYTHON3_ALIAS) -m venv venv || echo 'Python env already exists' + +--install_dependencies: requirements.txt + @# Help: Installs the dependencies from requirements.txt + -@echo 'Installing Dependencies...' + @$(PYTHON) -m pip install --upgrade pip + @$(PIP) install -r requirements.txt + +env: --create_virtual_env --install_dependencies + @# Help: Creates a virtual environment with python 3 if not already present + +clean_env: + @# Help: Clean the virtual env that was created + -@rm -rf venv + +clean_build: + @# Help: Cleans the build files + -@rm -rf build/ + -@rm -rf dist/ + -@rm -rf delphix-nr.spec + +clean: clean_build + @# Help: Cleans the pycache, coverage and build files + -@rm -rf __pycache__ + -@rm -f .coverage + -@rm -rf .pytest_cache + -@rm -rf tests/.pytest_cache + -@rm -rf tests/plugin_operations/.pytest_cache + -@rm -rf tests/CodeCoverage + -@rm -rf tests/Report.html + -@rm -rf .dvp-gen-output + +build: --check_env_exists clean_build + @# Help: Makes the delphix-nr build + $(VENV)/bin/pyinstaller --onefile src/main.py -n delphix-nr + -@echo 'Build is present at $(ROOT_DIR)/dist/delphix-nr' + $(call show_version) + + +# A hidden target +.hidden: +help: + @printf "%-20s %s\n" "Target" "Description" + @printf "%-20s %s\n" "------" "-----------" + @make -pqR : 2>/dev/null \ + | awk -v RS= -F: '/^# File/,/^# Finished Make data base/ {if ($$1 !~ "^[#.]") {print $$1}}' \ + | sort \ + | egrep -v -e '^[^[:alnum:]]' -e '^$@$$' \ + | xargs -I _ sh -c 'printf "%-20s " _; make _ -nB | (grep -i "^# Help:" || echo "") | tail -1 | sed "s/^# Help: //g"' diff --git a/README.md b/README.md index e9b8ad1..59b4137 100644 --- a/README.md +++ b/README.md @@ -1,91 +1,72 @@ -# Delphix Data Control Tower MultiCloud integration with New Relic +# Delphix Data Control Tower's Integration with New Relic -This project will allow you to send data from [Delphix Data Control Tower Multicloud](https://docs.delphix.com/dctmc) to [New Relic](https://newrelic.com/) as events. DCT Multicloud is a tool that will allow you to connect to all your Delphix engines on premises or in the cloud (AWS, Azure, Google Cloud, OCI and IBM) +This project will allow you to send data from [Delphix Data Control Tower (DCT)](https://delphix.document360.io/dct/docs) to [New Relic](https://newrelic.com/) through the Events API. This repository is one component of the Delphix Quickstart. You can learn more on the [Delphix Instant Observability page](https://newrelic.com/instant-observability/delphix). ![Screenshot](images/image2.png) ## Getting Started -These instructions will provide the code you need to extract data from DCT Multicloud and send it to New Relic. + +These instructions will provide the information you need to extract data from DCT and send it to New Relic. The Python script can run from any location with access to both DCT and New Relic. ### Prerequisites -It's assumed that you have a New Relic valid account and one or many [Delphix Engines registered in DCT Multicloud](https://docs.delphix.com/dctmc/connecting-a-delphix-engine). -DCT Multicloud will extract data from the Delphix Engines and we will use [New Relic Telemetry SDK](https://docs.newrelic.com/docs/telemetry-data-platform/ingest-apis/telemetry-sdks-report-custom-telemetry-data/) to send that data to New Relic. -For this project, we will use the [Python SDK](https://github.com/newrelic/newrelic-telemetry-sdk-python), however you can use any of the available SDKs in different languages. +* New Relic Account: [Sign Up](https://newrelic.com/signup) +* Delphix Data Control Tower (DCT) with one or more engines: [Data Control Tower Docs](https://delphix.document360.io/dct/docs) +* Python 3.7+: [Python Install](https://www.python.org/downloads) +* This [GitHub repository](https://github.com/delphix/dct-newrelic-integration) +
+

Supported Python Versions and OS

+
-### Installing +- MacOS - Python3.7 and Python3.8 +- Linux - Python3.7+ +- Windows - Python3.7+ -To push the data from Delphix DCT Multicloud to New Relic, we use the script ```dlpx_dct_to_nr.py```. +
+ +### Installing To use this script we have to do some steps first: -* Generate the [keys to connect to DCT Multicloud](https://docs.delphix.com/dctmc/authentication) +* Generate the [key to connect to DCT](https://docs.delphix.com/dctmc/authentication) * Generate the [New Relic access key](https://docs.newrelic.com/docs/apis/intro-apis/new-relic-api-keys/#ingest-license-key) +* Note the URL of the DCT instance VM -Once we have these keys we need to replace them in the script: -* In req_headers we replace the DCT Multicloud key -* In NEW_RELIC_INSERT_KEY we replace the New Relic access key +### Setup +The ```src/main.py``` script contains the logic to perform the data upload. However, you must do some configuration first. -This is the script: +* We need to supply the above gathered information using 3 environment variables + * DCT_HOST_URL + * DCT_API_KEY + * NEW_RELIC_INSERT_KEY +* Clone this repository +* Go inside the project directory - `cd dct-newrelic-integration` -``` -import os -import requests -import json -import sys -import time -from newrelic_telemetry_sdk import Event, EventClient -from requests.packages.urllib3.exceptions import InsecureRequestWarning - -requests.packages.urllib3.disable_warnings(InsecureRequestWarning) - -DLPX_TYPES= ["engines","sources","dsources","vdbs","environments"] -# -# Request Headers ... -# -req_headers = { - 'Authorization': 'apk 2.bnQDDx46Z4CDlIShLw2ZHElWLyKtsmZaBjbQPjui8LcQ3nELbkdEbQJSki6vmwLf' -} - -# -# Python session, also handles the cookies ... -# -session = requests.session() - -# -# Login ... -# -os.environ['NEW_RELIC_INSERT_KEY'] = "87a453b2efe4nd4df78b167e7ac457e076c7NRAL" -print ('') -for i in DLPX_TYPES: - - response = requests.get('https://localhost:443/v1/'+i, headers=req_headers, verify=False) - responsej = json.loads(response.text) - print("") - print("") - print("**********************************************************************************************************************************") - event_client = EventClient(os.environ["NEW_RELIC_INSERT_KEY"]) - NEWRELIC_TYPE="Delphix " + str(i) - print (NEWRELIC_TYPE) - for line in responsej['items']: - event = Event( - NEWRELIC_TYPE, line - ) - print (event) - response = event_client.send(event) - response.raise_for_status() - print("Event sent successfully!") - print("") - -print ('') -sys.exit(0) -``` +For Mac and Linux: +* Run command `make env` (This will create the virtual environment) +* Run command `make run` (This will run the script and push the data) + +For Windows: +* Check that python 3 is installed +* Create a virtual environment running `python -m venv venv` +* Activate the virtual environment by running `venv\Scripts\activate` +* Install the dependencies by running `pip install -r requirements.txt` +* Set Python path by - `set PYTHONPATH=.` +* Run the script using `python src\main.py` -On execution, this script will extract data from all the registered Delphix Engines for the following metrics: +Note: You may modify the Python script directly, but it is best practice to specify sensitive data through environment variables. + +
+ +In production, it is common to use a scheduler, such as a systemd, nohup, or wininit.exe, to ensure the script continually runs. For example, the following nohup command will run the script every N seconds based on the Interval provided in the `dct_nr_config.ini` file: +```nohup make run &``` + + +On each execution, this script will extract the following metrics from all registered Delphix engines: * Engines - Data extraction date, CPU Count, Storage, Memory, Engine Type, Version, etc. * Environments - Data extraction date, Status, Engine ID, Name, etc. @@ -93,12 +74,20 @@ On execution, this script will extract data from all the registered Delphix Engi * dSources - Data extraction date, dSource Creation Date, dSource Type, Version, Name, Status, Size, etc. * VDBs - Data extraction date, Database Type and Version, Creation Date, Group Name, Name, Parent ID, Size, Status, etc. -This script can be added to cron or any scheduler to run in any time interval. Once the data is available in New Relic, it can be used to be queried or to create dashboards. +More over we have `dct_nr_config.ini` file which can be used to configure +- Logging Level +- Interval (in seconds): This script keeps running and sleeps for `INTERVAL` seconds after sending the data once. +- Components we need from DCT APIs + +Once the data is available in New Relic, it can be used to be queried or to create dashboards. + + +## Data, Dashboards, and Alerts -## Data and Dashboards +Once the data is available within New Relic, you are free to leverage it as you wish through customized Dashboards and Alerts. Samples can be found as a part of the [Delphix Quickstart](https://newrelic.com/instant-observability/delphix). -This is how the raw data looks like in the [Query your data](https://docs.newrelic.com/docs/query-your-data/explore-query-data/get-started/introduction-querying-new-relic-data/#browse-data) window for the VDB metric: +As an example, this is how the raw data looks like in the [Query your data](https://docs.newrelic.com/docs/query-your-data/explore-query-data/get-started/introduction-querying-new-relic-data/#browse-data) window for the VDB metric: ![Screenshot](images/image1.png) @@ -115,7 +104,7 @@ SELECT MAX(data_storage_capacity)-MIN(data_storage_used) FROM `Delphix engines` ## Contributing -Please read [CONTRIBUTING.md](https://github.com/delphix/.github/blob/master/CONTRIBUTING.md) for details on our code of conduct, and the process for submitting pull requests to us. +This project is currently not accepting external contributions. ## Versioning @@ -123,9 +112,9 @@ Please read [CONTRIBUTING.md](https://github.com/delphix/.github/blob/master/CON We use [SemVer](http://semver.org/) for versioning. For the versions available, see the [tags on this repository](https://github.com/your/project/tags). -## Reporting Issues +## Reporting Issues and Questions -Issues should be reported in the GitHub repo's issue tab. Include a link to it. +Please report all issues and questions in the [GitHub issue tab](https://github.com/delphix/dct-newrelic-integration/issues) or [Delphix Community page](https://community.delphix.com/home). Please include a complete problem description, error logs if appropriate, and directions on how to reproduce. ## Statement of Support @@ -146,4 +135,3 @@ License See the License for the specific language governing permissions and limitations under the License. ``` -Copyright (c) 2014, 2016 by Delphix. All rights reserved. diff --git a/SUPPORT.md b/SUPPORT.md index 60e840d..ed92594 100644 --- a/SUPPORT.md +++ b/SUPPORT.md @@ -1,2 +1,12 @@ ## Support This software is provided as-is, without warranty of any kind or commercial support through Delphix. See the associated license for additional details. Questions, issues, feature requests, and contributions should be directed to the community as outlined in the [Delphix Community Guidelines](https://delphix.github.io/community-guidelines.html). + +
+

Supported Python Versions and OS

+
+ +- MacOS - Python3.7 and Python3.8 +- Linux - Python3.7+ +- Windows - Python3.7+ + +
\ No newline at end of file diff --git a/dct_nr_config.ini b/dct_nr_config.ini new file mode 100644 index 0000000..82cf47d --- /dev/null +++ b/dct_nr_config.ini @@ -0,0 +1,11 @@ +[COMPONENTS] +; This list holds the components for which +; we send the data to new relic +; refer to DCT API's for getting these components +monitor = management/engines, sources, dsources, vdbs, environments + +[INTERVAL] +seconds = 30 + +[LOGGING] +LEVEL = INFO diff --git a/dlpx_dct_to_nr.py b/dlpx_dct_to_nr.py deleted file mode 100644 index 504a9b3..0000000 --- a/dlpx_dct_to_nr.py +++ /dev/null @@ -1,51 +0,0 @@ -import os -import requests -import json -import sys -import time -from newrelic_telemetry_sdk import Event, EventClient -from requests.packages.urllib3.exceptions import InsecureRequestWarning - -requests.packages.urllib3.disable_warnings(InsecureRequestWarning) - -DLPX_TYPES= ["engines","sources","dsources","vdbs","environments"] -# -# Request Headers ... -# -req_headers = { - 'Authorization': 'apk 2.bnQDDx46Z4CDlIShLw2ZHElWLyKtsmZaBjbQPjui8LcQ3TELbkdEbQJSki6vmwLf' -} - -# -# Python session, also handles the cookies ... -# -session = requests.session() - -# -# Login ... -# -os.environ['NEW_RELIC_INSERT_KEY'] = "87a453b2efe4bd4df78b167e7ac457e076c7NRAL" -print ('') -for i in DLPX_TYPES: - - response = requests.get('https://localhost:443/v1/'+i, headers=req_headers, verify=False) - responsej = json.loads(response.text) - print("") - print("") - print("**********************************************************************************************************************************") - event_client = EventClient(os.environ["NEW_RELIC_INSERT_KEY"]) - NEWRELIC_TYPE="Delphix " + str(i) - print (NEWRELIC_TYPE) - for line in responsej['items']: - event = Event( - NEWRELIC_TYPE, line - ) - print (event) - response = event_client.send(event) - response.raise_for_status() - print("Event sent successfully!") - print("") - -print ('') -sys.exit(0) - diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..8015547 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,9 @@ +# +# Copyright (c) 2022 by Delphix. All rights reserved. +# + +newrelic-telemetry-sdk==0.4.3 +pyinstaller==5.6.2 +requests==2.28.1 +tenacity==8.1.0 +urllib3==1.26.13 \ No newline at end of file diff --git a/setup.py b/setup.py new file mode 100644 index 0000000..de15820 --- /dev/null +++ b/setup.py @@ -0,0 +1,44 @@ +# +# Copyright (c) 2022 by Delphix. All rights reserved. +# + +from setuptools import find_packages +from setuptools import setup +from src.main import VERSION + +# Get the long description from the README file +long_description = "This is an executable for the Delphix Integration for New Relic" + +setup( + version=VERSION, + packages=find_packages(), + classifiers=[ + "Development Status :: 5 - Production/Stable", + "Intended Audience :: Developers", + "Intended Audience :: Customers", + "Topic :: Software Development :: Libraries :: Python Modules", + "Topic :: Monitoring", + "Programming Language :: Python :: 3.7", + "Programming Language :: Python :: 3.8" + ], + install_requires=[ + "newrelic-telemetry-sdk==0.4.3", + "requests==2.28.1", + "tenacity==8.1.0", + ], + # Format is mypkg.mymodule:the_function' + entry_points=""" + [console_scripts] + nr=src.main:run + """, + author="Delphix Engineering", + keywords="new-relic, delphix, dct", # noqa + license="Apache 2", + description="Delphix Integration for New Relic", + dependency_links=[], + name="delphix-nr", + long_description_content_type="text/markdown", + long_description=long_description, + include_package_data=True, + project_urls={}, # Optional +) diff --git a/src/__init__.py b/src/__init__.py new file mode 100644 index 0000000..f3f964d --- /dev/null +++ b/src/__init__.py @@ -0,0 +1,5 @@ +# +# Copyright (c) 2022 by Delphix. All rights reserved. +# + +VERSION = "0.1.0" diff --git a/src/main.py b/src/main.py new file mode 100644 index 0000000..6ac108c --- /dev/null +++ b/src/main.py @@ -0,0 +1,161 @@ +# +# Copyright (c) 2022 by Delphix. All rights reserved. +# + +""" +This python script takes care of sending the data +from DCT APIs for a set of components to New relic. +It can be run using Python 3.8 +""" +import datetime +import os.path +import sys +import time +import traceback + +import urllib3 +import newrelic_telemetry_sdk.client +import requests +import logging +import configparser +import tenacity + +from newrelic_telemetry_sdk import Event, EventClient +from urllib3.exceptions import InsecureRequestWarning +from src import VERSION + +urllib3.disable_warnings(InsecureRequestWarning) + +config_file = "dct_nr_config.ini" +config = configparser.ConfigParser() + +if not os.path.exists(config_file): + raise Exception("Config file is required." + " Please add dct_nr_config.ini in the current directory") + +config.read_file(open(config_file)) +config_log_level = config['LOGGING']['LEVEL'] +INTERVAL = int(config['INTERVAL']['seconds']) + +formatter = '%(asctime)s - %(name)s - %(levelname)s - %(message)s' +logging_levels = { + 'INFO': logging.INFO, + 'DEBUG': logging.DEBUG, + 'WARN': logging.WARNING, + 'ERROR': logging.ERROR +} +logging.basicConfig(level=logging_levels[config_log_level], format=formatter) + + +DCT_HOST_URL = os.environ.get('DCT_HOST_URL') +DCT_API_KEY = os.environ.get('DCT_API_KEY') +NEW_RELIC_INSERT_KEY = os.environ.get('NEW_RELIC_INSERT_KEY') + +if (DCT_HOST_URL and DCT_API_KEY and NEW_RELIC_INSERT_KEY) is None: + raise Exception("Please add the below keys as environment variables: " + "\n1. DCT_HOST_URL" + "\n2. DCT_API_KEY" + "\n3. NEW_RELIC_INSERT_KEY") + +DCT_API_URL = f'https://{DCT_HOST_URL}/v2/' +QUERY = '?limit=1000' +DLPX_TYPES = [i.strip() for i in config['COMPONENTS']['monitor'].split(',')] + +# Request Headers ... +req_headers = { + 'Authorization': DCT_API_KEY # noqa +} + +# Python session, also handles the cookies ... +session = requests.session() + + +@tenacity.retry( + retry=tenacity.retry_if_exception_type( + newrelic_telemetry_sdk.client.HTTPError + ), + stop=tenacity.stop_after_attempt(5), + wait=tenacity.wait_fixed(5), + before_sleep=tenacity.before_sleep_log(logging, logging.INFO), +) +def send_event(event): + logging.debug(event) + event_client = EventClient(NEW_RELIC_INSERT_KEY) + response = event_client.send(event) + response.raise_for_status() + logging.debug("Event sent successfully!\n") + + +@tenacity.retry( + retry=tenacity.retry_if_exception_type( + AssertionError + ), + stop=tenacity.stop_after_attempt(5), + wait=tenacity.wait_fixed(5), + before_sleep=tenacity.before_sleep_log(logging, logging.DEBUG), +) +def get_data_from_dct(component, query=QUERY): + url = f"{DCT_API_URL}{component}{query}" + response = requests.get(url, headers=req_headers, verify=False) + logging.debug("DCT API - %s responded with %s", url, response.status_code) + assert response.status_code == 200 + response_json = response.json() + records = response_json['response_metadata']['total'] + logging.debug(f"Get {component} returned {records} records") + + # handle pagination + if 'next_cursor' in response_json['response_metadata']: + logging.debug('Response is paginated getting the next page') + next_cursor = response_json['response_metadata']['next_cursor'] + next_page = get_data_from_dct( + component, query=QUERY+'&cursor={}'.format(next_cursor) + ) + response_json['items'].extend(next_page['items']) + + return response_json + + +def push_data(): + while True: + logging.info('*' * 70) + logging.info("Pushing Data to New Relic at %s", + datetime.datetime.now()) + logging.info('*' * 70) + for i in DLPX_TYPES: + try: + logging.debug("Getting Data from DCT for %s ", i) + response_json = get_data_from_dct(i) + + new_relic_type = "Delphix " + str(i).split('/')[-1] + logging.info("Sending Data for %s \n", new_relic_type) + for line in response_json['items']: + event = Event(new_relic_type, line) + send_event(event) + except Exception as e: + logging.error("Exception occurred while collecting or" + " sending data for component - %s", i) + logging.error(e) + logging.error(traceback.format_exc()) + + logging.info(f"Sleeping for %s seconds before pushing again", INTERVAL) + logging.info('-' * 70) + time.sleep(INTERVAL) + + +def run(): + if len(sys.argv) > 1: + if sys.argv[1] == "--version": + print(VERSION) + else: + logging.info('#' * 70) + logging.info("Staring to Push Data to New Relic") + logging.info('#' * 70) + logging.debug(f"{DCT_HOST_URL=}") + logging.debug(f"{DLPX_TYPES=}") + logging.debug(f"{req_headers=}") + logging.debug(f"NEW_RELIC_INSERT_KEY = {NEW_RELIC_INSERT_KEY}") + push_data() + + +if __name__ == '__main__': + run()