You are browsing a read-only backup copy of Wikitech. The live site can be found at wikitech.wikimedia.org

User:Nskaggs: Difference between revisions

From Wikitech-static
Jump to navigation Jump to search
imported>Nskaggs
No edit summary
imported>Nskaggs
(Add more random python snippets)
Line 1: Line 1:
An enhanced fork of [[User:Arturo Borrero Gonzalez#wmcs-netbox-list.py|Arturo's netbox-list.py]] with more fields gathered.


===wmcs-netbox-list.py===
===wmcs-netbox-list.py===


I use this script to fetch & list WMCS server info from [[Netbox]] and generate a CSV (to later import into a spreadsheet):
An enhanced fork of [[User:Arturo Borrero Gonzalez#wmcs-netbox-list.py|Arturo's netbox-list.py]] with more fields gathered. Fetch & list WMCS server info from [[Netbox]] and generate a CSV (to later import into a spreadsheet):




Line 126: Line 125:
if __name__ == "__main__":
if __name__ == "__main__":
     main()
     main()
</syntaxhighlight>
{{Collapse bottom}}
===wmcs-phab.py===
Proof of concept to pull ticket information from phabricator to csv
{{Collapse top|wmcs-phab.py}}
<syntaxhighlight lang="python">
#!/usr/bin/env python3
import argparse
from phabricator import Phabricator
import time
class Config:
    """ configuration for this script.
        """
    def __init__(self, api_token, phab_url, project):
        self.api_token = api_token
        if not phab_url:
            self.phab_url = "https://phabricator.wikimedia.org/api/"
        else:
            self.phab_url = phab_url
        if not project:
            #cloud services kanban
            self.project = "2774"
        else:
            self.project = project
        self.phab = Phabricator(host=self.phab_url, token=self.api_token)
def print_result_header():
    print(
        "id, dateCreated, dateModified, status, priority, author, owner, title"
    )
def print_result(result):
    print(
        "{}, {}, {}, {}, {}, {}, {}, {}".format(
            result["id"],
            time.strftime("%Y-%m-%d", time.gmtime(result["created"])),
            time.strftime("%Y-%m-%d", time.gmtime(result["modified"])),
            #result["created"],
            #result["modified"],
            result["status"],
            result["priority"],
            result["author"],
            result["owner"],
            result["title"]
           
        )
    )
def open_phab_tasks(config):
    project = config.phab.project.query(ids=[config.project])
    p_pid = list(project['data'].keys())[0]
    tasks = config.phab.maniphest.query(projectPHIDs=[p_pid],status='status-open')
    results = []
    for key, task in tasks.items():
        a_pid = task['authorPHID']
        o_pid = task['ownerPHID']
        author = config.phab.user.query(phids=[a_pid])[0]["userName"]
        if config.phab.user.query(phids=[o_pid]):
            owner = config.phab.user.query(phids=[o_pid])[0]["userName"]
        else:
            owner = "None"
        result = {}
        result['id'] = task['id']
        result['created'] = int(task['dateCreated'])
        result['modified'] = int(task['dateModified'])
        result['status'] = task['status']
        result['priority'] = task['priority']
        result['author'] = author
        result['owner'] = owner
        result['title'] = task['title'].replace(',','')
        results.append(result)
    return results
def main():
    parser = argparse.ArgumentParser(
        description="Utility to information from Phabricator"
    )
    parser.add_argument("--token", action="store", help="Phabricator API token")
    parser.add_argument("--url", action="store", help="Phabricator URL")
    parser.add_argument("--project", action="store", help="Project ID")
    args = parser.parse_args()
    config = Config(args.token, args.url, args.project)
   
    print_result_header()
    results = open_phab_tasks(config)
    for result in results:
        print_result(result)
if __name__ == "__main__":
    main()
</syntaxhighlight>
{{Collapse bottom}}
===wmcs-openstack-usage.py===
Quick script used to help generate raw data for cost estimations of WMCS cloud usage
{{Collapse top|wmcs-openstack-usage.py}}
<syntaxhighlight lang="python">
#!/usr/bin/env python3
import io
import subprocess
def main():
    startdate = '2021-08-21'
    enddate = '2021-09-21'
    fu = open('usage.csv', 'w')
    fu.write(f'Project,CPU Hours,Disk GB-Hours,RAM MB-Hours,Servers\n')
    fl = open('limits.csv', 'w')
    fl.write('Project, maxInstances, Instances, maxCPU, CPU, maxRAM, RAM (MB))\n')
    fq = open('quota.csv', 'w')
    fq.write('Project, gigabytes\n')
    fr = open('resource.csv', 'w')
    fr.write('Project, VCPU, MEMORY_MD, DISK_G\n')
    # get list of projects
    output = subprocess.run(['sudo', 'wmcs-openstack','project','list', '-f', 'value', '-c', 'ID'], stdout=subprocess.PIPE).stdout.decode('utf-8')
    projects = output.split('\n')[:-1]
    for project in projects:
        # gather usage stats
        output = subprocess.run(['sudo', 'wmcs-openstack','usage','show', '--project', project, '--print-empty', '-f', 'value', '--start', startdate, '--end', enddate], stdout=subprocess.PIPE).stdout.decode('utf-8')
        usage = output.split('\n')[:-1]
        #parse stats and output as csv
        fu.write(f'{project},{usage[0]},{usage[1]},{usage[2]},{usage[3]}\n')
        # gather limit stats
        output = subprocess.run(['sudo', 'wmcs-openstack','limits','show', '--project', project, '--absolute', '-f', 'value'], stdout=subprocess.PIPE).stdout.decode('utf-8')
        limits = output.split('\n')
        # parse out resource usage and limits
        for line in limits:
            attr = line.split(' ')[0]
            if attr == 'maxTotalInstances':
                maxinstances = line.split(' ')[1]
            elif attr == 'totalInstancesUsed':
                instances = line.split(' ')[1]
            elif attr == 'maxTotalCores':
                maxcpu = line.split(' ')[1]
            elif attr == 'totalCoresUsed':
                cpu = line.split(' ')[1]
            elif attr == 'maxTotalRAMSize':
                maxram = line.split(' ')[1]
            elif attr == 'totalRAMUsed':
                ram = line.split(' ')[1]
        #parse stats and output as csv
        fl.write(f'{project},{maxinstances},{instances},{maxcpu},{cpu},{maxram},{ram},\n')
        # in order to get disk usage, we need to check other calls
        # gather maxdisk (cinder)
        ps = subprocess.run(['sudo', 'wmcs-openstack','quota','show', project, '-f', 'table'], stdout=subprocess.PIPE)
        pipe = subprocess.run(['grep', '| gigabytes            |'], input=ps.stdout, stdout=subprocess.PIPE)
        output = subprocess.run(['cut', '-d', '|', '-f', '3'], input=pipe.stdout, stdout=subprocess.PIPE).stdout.decode('utf-8')
        diskquota = output.strip()
        fq.write(f'{project},{diskquota},\n')
        # gather disk used (cinder)
        output = subprocess.run(['sudo', 'wmcs-openstack','resource','usage', 'show', '--os-placement-api-version', '1.9', project, '-f', 'value'], stdout=subprocess.PIPE).stdout.decode('utf-8')
        resources = output.split('\n')
        for line in resources:
            attr = line.split(' ')[0]
            if attr == 'VCPU':
                vcpu = line.split(' ')[1]
            elif attr == 'MEMORY_MB':
                memory_mb = line.split(' ')[1]
            elif attr == 'DISK_GB':
                disk_gb = line.split(' ')[1]
        fr.write(f'{project},{vcpu},{memory_mb},{disk_gb}\n')
    fu.close()
    fl.close()
    fq.close()
    fr.close()
    fh = open('hosts.csv', 'w')
    fh.write(f'Host,Summary,CPU,Memory MB,Disk GB\n')
    fp = open('projects.csv', 'w')
    fp.write(f'Project,Host,CPU,Memory MB,Disk GB\n')
    # get list of hosts and show
    output = subprocess.run(['sudo', 'wmcs-openstack','host','list', '-f', 'value', '-c', 'Host Name', '--zone', 'nova'], stdout=subprocess.PIPE).stdout.decode('utf-8')
    hosts = output.split('\n')[:-1]
    # seperate host totals from project stats
    for host in hosts:
        output = subprocess.run(['sudo', 'wmcs-openstack','host','show', host, '-f', 'value'], stdout=subprocess.PIPE).stdout.decode('utf-8')
        stats = output.split('\n')[:-1]
        host_lines = stats[0:3]
        project_lines = stats[3:]
        # write out host info
        for line in host_lines:
            entry = line.split(' ')
            fh.write(f'{entry[0]},{entry[1]},{entry[2]},{entry[3]},{entry[4]}\n')
        # write out project info
        for line in project_lines:
            entry = line.split(' ')
            fp.write(f'{entry[1]},{entry[0]},{entry[2]},{entry[3]},{entry[4]}\n')
    fh.close()
    fp.close()
main()


</syntaxhighlight>
</syntaxhighlight>
{{Collapse bottom}}
{{Collapse bottom}}

Revision as of 16:20, 25 January 2022

wmcs-netbox-list.py

An enhanced fork of Arturo's netbox-list.py with more fields gathered. Fetch & list WMCS server info from Netbox and generate a CSV (to later import into a spreadsheet):


wmcs-netbox-list.py
The following content has been placed in a collapsed box for improved usability. 
#!/usr/bin/env python3

import requests
import argparse


class Config:
    """ configuration for this script.
        """

    def __init__(self, api_token, netbox_url):
        self.api_token = api_token
        if not netbox_url:
            self.netbox_url = "https://netbox.wikimedia.org"
        else:
            self.netbox_url = netbox_url


def generate_netbox_link(config, id):
    return "{}/dcim/devices/{}/".format(config.netbox_url, id)


def print_result_header():
    print(
        "name, purchase date, ticket, status, manufacturer, model, serial, asset tag, rack, position, IPv4, vlan, site, netbox link"
    )


def print_result(config, result):
    netbox_link = generate_netbox_link(config, result["id"])
    if not result["primary_ip4"]:
        result["primary_ip4"] = {}
        result["primary_ip4"]["address"] = "None"
        vlan = "None"
    else:
        vlan = request_query_vlan(config, result)
    if not result["rack"]:
        result["rack"] = {}
        result["rack"]["name"] = "None"
    print(
        "{}, {}, {}, {}, {}, {}, {}, {}, {}, {}, {}, {}, {}, {}".format(
            result["name"],
            result["custom_fields"]["purchase_date"],
            result["custom_fields"]["ticket"],
            result["status"]["value"],
            result["device_type"]["manufacturer"]["name"],
            result["device_type"]["model"],
            result["serial"],
            result["asset_tag"],
            result["rack"]["name"],
            result["position"],
            result["primary_ip4"]["address"],
            vlan,
            result["site"]["slug"],
            netbox_link,
        )
    )


def request_query_list(config, query):
    request_url = "{}/api/dcim/devices/?q={}".format(config.netbox_url, query)
    request_headers = {"Authorization": "Token {}".format(config.api_token)}
    return requests.get(url=request_url, headers=request_headers)


def request_query_vlan(config, result):
    ipaddr = result["primary_ip4"]["id"]
    request_url = "{}/api/ipam/ip-addresses/{}".format(config.netbox_url, ipaddr)
    request_headers = {"Authorization": "Token {}".format(config.api_token)}
    ip_result = requests.get(url=request_url, headers=request_headers)
    
    if not ip_result.json()["assigned_object_id"]:
        return "None"
    interface = ip_result.json()["assigned_object_id"]
    request_url = "{}/api/dcim/interfaces/{}".format(config.netbox_url, interface)
    request_headers = {"Authorization": "Token {}".format(config.api_token)}
    interface_result = requests.get(url=request_url, headers=request_headers)
    
    if not interface_result.json()["connected_endpoint"]:
        return "None"
    endpoint = interface_result.json()["connected_endpoint"]["id"]
    request_url = "{}/api/dcim/interfaces/{}".format(config.netbox_url, endpoint)
    request_headers = {"Authorization": "Token {}".format(config.api_token)}
    endpoint_result = requests.get(url=request_url, headers=request_headers)

    if endpoint_result.json()["untagged_vlan"]:
        return endpoint_result.json()["untagged_vlan"]["name"]
    else:
        return "None"


def main():
    parser = argparse.ArgumentParser(
        description="Utility to fetch and list WMCS servers from Netbox"
    )
    parser.add_argument("--token", action="store", help="Netbox API token")
    parser.add_argument("--url", action="store", help="Netbox URL")
    args = parser.parse_args()

    config = Config(args.token, args.url)

    print_result_header()
    r = request_query_list(config, "cloud")
    for result in r.json()["results"]:
        print_result(config, result)

    r = request_query_list(config, "lab")
    for result in r.json()["results"]:
        print_result(config, result)

    r = request_query_list(config, "labtest")
    for result in r.json()["results"]:
        print_result(config, result)


if __name__ == "__main__":
    main()
The above content has been placed in a collapsed box for improved usability.

wmcs-phab.py

Proof of concept to pull ticket information from phabricator to csv


wmcs-phab.py
The following content has been placed in a collapsed box for improved usability. 
#!/usr/bin/env python3

import argparse
from phabricator import Phabricator
import time


class Config:
    """ configuration for this script.
        """

    def __init__(self, api_token, phab_url, project):
        self.api_token = api_token
        if not phab_url:
            self.phab_url = "https://phabricator.wikimedia.org/api/"
        else:
            self.phab_url = phab_url
        if not project:
            #cloud services kanban
            self.project = "2774"
        else:
            self.project = project

        self.phab = Phabricator(host=self.phab_url, token=self.api_token)

def print_result_header():
    print(
        "id, dateCreated, dateModified, status, priority, author, owner, title"
    )

def print_result(result):
    print(
        "{}, {}, {}, {}, {}, {}, {}, {}".format(
            result["id"],
            time.strftime("%Y-%m-%d", time.gmtime(result["created"])),
            time.strftime("%Y-%m-%d", time.gmtime(result["modified"])),
            #result["created"],
            #result["modified"],
            result["status"],
            result["priority"],
            result["author"],
            result["owner"],
            result["title"]
            
        )
    )


def open_phab_tasks(config):
    project = config.phab.project.query(ids=[config.project])
    p_pid = list(project['data'].keys())[0]
    tasks = config.phab.maniphest.query(projectPHIDs=[p_pid],status='status-open')

    results = []
    for key, task in tasks.items():
        a_pid = task['authorPHID']
        o_pid = task['ownerPHID']
        author = config.phab.user.query(phids=[a_pid])[0]["userName"]
        if config.phab.user.query(phids=[o_pid]): 
            owner = config.phab.user.query(phids=[o_pid])[0]["userName"]
        else:
            owner = "None"

        result = {}
        result['id'] = task['id']
        result['created'] = int(task['dateCreated'])
        result['modified'] = int(task['dateModified'])
        result['status'] = task['status']
        result['priority'] = task['priority']
        result['author'] = author
        result['owner'] = owner
        result['title'] = task['title'].replace(',','')
        results.append(result)

    return results


def main():
    parser = argparse.ArgumentParser(
        description="Utility to information from Phabricator"
    )
    parser.add_argument("--token", action="store", help="Phabricator API token")
    parser.add_argument("--url", action="store", help="Phabricator URL")
    parser.add_argument("--project", action="store", help="Project ID")
    args = parser.parse_args()

    config = Config(args.token, args.url, args.project)
    
    print_result_header()
    results = open_phab_tasks(config)
    for result in results:
        print_result(result)

if __name__ == "__main__":
    main()
The above content has been placed in a collapsed box for improved usability.


wmcs-openstack-usage.py

Quick script used to help generate raw data for cost estimations of WMCS cloud usage


wmcs-openstack-usage.py
The following content has been placed in a collapsed box for improved usability. 
#!/usr/bin/env python3

import io
import subprocess


def main():
    startdate = '2021-08-21'
    enddate = '2021-09-21'

    fu = open('usage.csv', 'w')
    fu.write(f'Project,CPU Hours,Disk GB-Hours,RAM MB-Hours,Servers\n')

    fl = open('limits.csv', 'w')
    fl.write('Project, maxInstances, Instances, maxCPU, CPU, maxRAM, RAM (MB))\n')

    fq = open('quota.csv', 'w')
    fq.write('Project, gigabytes\n')

    fr = open('resource.csv', 'w')
    fr.write('Project, VCPU, MEMORY_MD, DISK_G\n')

    # get list of projects
    output = subprocess.run(['sudo', 'wmcs-openstack','project','list', '-f', 'value', '-c', 'ID'], stdout=subprocess.PIPE).stdout.decode('utf-8')
    projects = output.split('\n')[:-1]

    for project in projects:
        # gather usage stats
        output = subprocess.run(['sudo', 'wmcs-openstack','usage','show', '--project', project, '--print-empty', '-f', 'value', '--start', startdate, '--end', enddate], stdout=subprocess.PIPE).stdout.decode('utf-8')
        usage = output.split('\n')[:-1]
        #parse stats and output as csv
        fu.write(f'{project},{usage[0]},{usage[1]},{usage[2]},{usage[3]}\n')

        # gather limit stats
        output = subprocess.run(['sudo', 'wmcs-openstack','limits','show', '--project', project, '--absolute', '-f', 'value'], stdout=subprocess.PIPE).stdout.decode('utf-8')
        limits = output.split('\n')

        # parse out resource usage and limits
        for line in limits:
            attr = line.split(' ')[0]
            if attr == 'maxTotalInstances':
                maxinstances = line.split(' ')[1]
            elif attr == 'totalInstancesUsed':
                instances = line.split(' ')[1]
            elif attr == 'maxTotalCores':
                maxcpu = line.split(' ')[1]
            elif attr == 'totalCoresUsed':
                cpu = line.split(' ')[1]
            elif attr == 'maxTotalRAMSize':
                maxram = line.split(' ')[1]
            elif attr == 'totalRAMUsed':
                ram = line.split(' ')[1]

        #parse stats and output as csv
        fl.write(f'{project},{maxinstances},{instances},{maxcpu},{cpu},{maxram},{ram},\n')

        # in order to get disk usage, we need to check other calls
        # gather maxdisk (cinder)
        ps = subprocess.run(['sudo', 'wmcs-openstack','quota','show', project, '-f', 'table'], stdout=subprocess.PIPE)
        pipe = subprocess.run(['grep', '| gigabytes             |'], input=ps.stdout, stdout=subprocess.PIPE)
        output = subprocess.run(['cut', '-d', '|', '-f', '3'], input=pipe.stdout, stdout=subprocess.PIPE).stdout.decode('utf-8')
        diskquota = output.strip()
        fq.write(f'{project},{diskquota},\n')

        # gather disk used (cinder)
        output = subprocess.run(['sudo', 'wmcs-openstack','resource','usage', 'show', '--os-placement-api-version', '1.9', project, '-f', 'value'], stdout=subprocess.PIPE).stdout.decode('utf-8')
        resources = output.split('\n')
 
        for line in resources:
            attr = line.split(' ')[0]
            if attr == 'VCPU':
                vcpu = line.split(' ')[1]
            elif attr == 'MEMORY_MB':
                memory_mb = line.split(' ')[1]
            elif attr == 'DISK_GB':
                disk_gb = line.split(' ')[1]

        fr.write(f'{project},{vcpu},{memory_mb},{disk_gb}\n')

    fu.close()
    fl.close()
    fq.close()
    fr.close()

    fh = open('hosts.csv', 'w')
    fh.write(f'Host,Summary,CPU,Memory MB,Disk GB\n')

    fp = open('projects.csv', 'w')
    fp.write(f'Project,Host,CPU,Memory MB,Disk GB\n')

    # get list of hosts and show
    output = subprocess.run(['sudo', 'wmcs-openstack','host','list', '-f', 'value', '-c', 'Host Name', '--zone', 'nova'], stdout=subprocess.PIPE).stdout.decode('utf-8')
    hosts = output.split('\n')[:-1]

    # seperate host totals from project stats
    for host in hosts:
        output = subprocess.run(['sudo', 'wmcs-openstack','host','show', host, '-f', 'value'], stdout=subprocess.PIPE).stdout.decode('utf-8')
        stats = output.split('\n')[:-1]
        host_lines = stats[0:3]
        project_lines = stats[3:]

        # write out host info
        for line in host_lines:
            entry = line.split(' ')
            fh.write(f'{entry[0]},{entry[1]},{entry[2]},{entry[3]},{entry[4]}\n')

        # write out project info
        for line in project_lines:
            entry = line.split(' ')
            fp.write(f'{entry[1]},{entry[0]},{entry[2]},{entry[3]},{entry[4]}\n')
    fh.close()
    fp.close()

main()
The above content has been placed in a collapsed box for improved usability.
Retrieved from "https://wikitech-static.wikimedia.org/w/index.php?title=User:Nskaggs&oldid=586146"