You are browsing a read-only backup copy of Wikitech. The live site can be found at wikitech.wikimedia.org

User:Nskaggs: Difference between revisions

From Wikitech-static
Jump to navigation Jump to search
imported>Nskaggs
No edit summary
imported>Nskaggs
(Add more random python snippets)
Line 1: Line 1:
An enhanced fork of [[User:Arturo Borrero Gonzalez#wmcs-netbox-list.py|Arturo's netbox-list.py]] with more fields gathered.


===wmcs-netbox-list.py===
===wmcs-netbox-list.py===


I use this script to fetch & list WMCS server info from [[Netbox]] and generate a CSV (to later import into a spreadsheet):
An enhanced fork of [[User:Arturo Borrero Gonzalez#wmcs-netbox-list.py|Arturo's netbox-list.py]] with more fields gathered. Fetch & list WMCS server info from [[Netbox]] and generate a CSV (to later import into a spreadsheet):




Line 126: Line 125:
if __name__ == "__main__":
if __name__ == "__main__":
     main()
     main()
</syntaxhighlight>
{{Collapse bottom}}
===wmcs-phab.py===
Proof of concept to pull ticket information from phabricator to csv
{{Collapse top|wmcs-phab.py}}
<syntaxhighlight lang="python">
#!/usr/bin/env python3
import argparse
from phabricator import Phabricator
import time
class Config:
    """ configuration for this script.
        """
    def __init__(self, api_token, phab_url, project):
        self.api_token = api_token
        if not phab_url:
            self.phab_url = "https://phabricator.wikimedia.org/api/"
        else:
            self.phab_url = phab_url
        if not project:
            #cloud services kanban
            self.project = "2774"
        else:
            self.project = project
        self.phab = Phabricator(host=self.phab_url, token=self.api_token)
def print_result_header():
    print(
        "id, dateCreated, dateModified, status, priority, author, owner, title"
    )
def print_result(result):
    print(
        "{}, {}, {}, {}, {}, {}, {}, {}".format(
            result["id"],
            time.strftime("%Y-%m-%d", time.gmtime(result["created"])),
            time.strftime("%Y-%m-%d", time.gmtime(result["modified"])),
            #result["created"],
            #result["modified"],
            result["status"],
            result["priority"],
            result["author"],
            result["owner"],
            result["title"]
           
        )
    )
def open_phab_tasks(config):
    project = config.phab.project.query(ids=[config.project])
    p_pid = list(project['data'].keys())[0]
    tasks = config.phab.maniphest.query(projectPHIDs=[p_pid],status='status-open')
    results = []
    for key, task in tasks.items():
        a_pid = task['authorPHID']
        o_pid = task['ownerPHID']
        author = config.phab.user.query(phids=[a_pid])[0]["userName"]
        if config.phab.user.query(phids=[o_pid]):
            owner = config.phab.user.query(phids=[o_pid])[0]["userName"]
        else:
            owner = "None"
        result = {}
        result['id'] = task['id']
        result['created'] = int(task['dateCreated'])
        result['modified'] = int(task['dateModified'])
        result['status'] = task['status']
        result['priority'] = task['priority']
        result['author'] = author
        result['owner'] = owner
        result['title'] = task['title'].replace(',','')
        results.append(result)
    return results
def main():
    parser = argparse.ArgumentParser(
        description="Utility to information from Phabricator"
    )
    parser.add_argument("--token", action="store", help="Phabricator API token")
    parser.add_argument("--url", action="store", help="Phabricator URL")
    parser.add_argument("--project", action="store", help="Project ID")
    args = parser.parse_args()
    config = Config(args.token, args.url, args.project)
   
    print_result_header()
    results = open_phab_tasks(config)
    for result in results:
        print_result(result)
if __name__ == "__main__":
    main()
</syntaxhighlight>
{{Collapse bottom}}
===wmcs-openstack-usage.py===
Quick script used to help generate raw data for cost estimations of WMCS cloud usage
{{Collapse top|wmcs-openstack-usage.py}}
<syntaxhighlight lang="python">
#!/usr/bin/env python3
import io
import subprocess
def main():
    startdate = '2021-08-21'
    enddate = '2021-09-21'
    fu = open('usage.csv', 'w')
    fu.write(f'Project,CPU Hours,Disk GB-Hours,RAM MB-Hours,Servers\n')
    fl = open('limits.csv', 'w')
    fl.write('Project, maxInstances, Instances, maxCPU, CPU, maxRAM, RAM (MB))\n')
    fq = open('quota.csv', 'w')
    fq.write('Project, gigabytes\n')
    fr = open('resource.csv', 'w')
    fr.write('Project, VCPU, MEMORY_MD, DISK_G\n')
    # get list of projects
    output = subprocess.run(['sudo', 'wmcs-openstack','project','list', '-f', 'value', '-c', 'ID'], stdout=subprocess.PIPE).stdout.decode('utf-8')
    projects = output.split('\n')[:-1]
    for project in projects:
        # gather usage stats
        output = subprocess.run(['sudo', 'wmcs-openstack','usage','show', '--project', project, '--print-empty', '-f', 'value', '--start', startdate, '--end', enddate], stdout=subprocess.PIPE).stdout.decode('utf-8')
        usage = output.split('\n')[:-1]
        #parse stats and output as csv
        fu.write(f'{project},{usage[0]},{usage[1]},{usage[2]},{usage[3]}\n')
        # gather limit stats
        output = subprocess.run(['sudo', 'wmcs-openstack','limits','show', '--project', project, '--absolute', '-f', 'value'], stdout=subprocess.PIPE).stdout.decode('utf-8')
        limits = output.split('\n')
        # parse out resource usage and limits
        for line in limits:
            attr = line.split(' ')[0]
            if attr == 'maxTotalInstances':
                maxinstances = line.split(' ')[1]
            elif attr == 'totalInstancesUsed':
                instances = line.split(' ')[1]
            elif attr == 'maxTotalCores':
                maxcpu = line.split(' ')[1]
            elif attr == 'totalCoresUsed':
                cpu = line.split(' ')[1]
            elif attr == 'maxTotalRAMSize':
                maxram = line.split(' ')[1]
            elif attr == 'totalRAMUsed':
                ram = line.split(' ')[1]
        #parse stats and output as csv
        fl.write(f'{project},{maxinstances},{instances},{maxcpu},{cpu},{maxram},{ram},\n')
        # in order to get disk usage, we need to check other calls
        # gather maxdisk (cinder)
        ps = subprocess.run(['sudo', 'wmcs-openstack','quota','show', project, '-f', 'table'], stdout=subprocess.PIPE)
        pipe = subprocess.run(['grep', '| gigabytes            |'], input=ps.stdout, stdout=subprocess.PIPE)
        output = subprocess.run(['cut', '-d', '|', '-f', '3'], input=pipe.stdout, stdout=subprocess.PIPE).stdout.decode('utf-8')
        diskquota = output.strip()
        fq.write(f'{project},{diskquota},\n')
        # gather disk used (cinder)
        output = subprocess.run(['sudo', 'wmcs-openstack','resource','usage', 'show', '--os-placement-api-version', '1.9', project, '-f', 'value'], stdout=subprocess.PIPE).stdout.decode('utf-8')
        resources = output.split('\n')
        for line in resources:
            attr = line.split(' ')[0]
            if attr == 'VCPU':
                vcpu = line.split(' ')[1]
            elif attr == 'MEMORY_MB':
                memory_mb = line.split(' ')[1]
            elif attr == 'DISK_GB':
                disk_gb = line.split(' ')[1]
        fr.write(f'{project},{vcpu},{memory_mb},{disk_gb}\n')
    fu.close()
    fl.close()
    fq.close()
    fr.close()
    fh = open('hosts.csv', 'w')
    fh.write(f'Host,Summary,CPU,Memory MB,Disk GB\n')
    fp = open('projects.csv', 'w')
    fp.write(f'Project,Host,CPU,Memory MB,Disk GB\n')
    # get list of hosts and show
    output = subprocess.run(['sudo', 'wmcs-openstack','host','list', '-f', 'value', '-c', 'Host Name', '--zone', 'nova'], stdout=subprocess.PIPE).stdout.decode('utf-8')
    hosts = output.split('\n')[:-1]
    # seperate host totals from project stats
    for host in hosts:
        output = subprocess.run(['sudo', 'wmcs-openstack','host','show', host, '-f', 'value'], stdout=subprocess.PIPE).stdout.decode('utf-8')
        stats = output.split('\n')[:-1]
        host_lines = stats[0:3]
        project_lines = stats[3:]
        # write out host info
        for line in host_lines:
            entry = line.split(' ')
            fh.write(f'{entry[0]},{entry[1]},{entry[2]},{entry[3]},{entry[4]}\n')
        # write out project info
        for line in project_lines:
            entry = line.split(' ')
            fp.write(f'{entry[1]},{entry[0]},{entry[2]},{entry[3]},{entry[4]}\n')
    fh.close()
    fp.close()
main()


</syntaxhighlight>
</syntaxhighlight>
{{Collapse bottom}}
{{Collapse bottom}}

Revision as of 16:20, 25 January 2022

wmcs-netbox-list.py

An enhanced fork of Arturo's netbox-list.py with more fields gathered. Fetch & list WMCS server info from Netbox and generate a CSV (to later import into a spreadsheet):


wmcs-phab.py

Proof of concept to pull ticket information from phabricator to csv



wmcs-openstack-usage.py

Quick script used to help generate raw data for cost estimations of WMCS cloud usage