BackupService/app.py

import os
from datetime import datetime, timedelta
import ftplib
from subprocess import Popen, PIPE
import shutil

'''
 TODO
 mysql bulk BDEGT database backup to dev testing db
 zip mongodb backup and put on over FTP
'''

DB_LIST = ['BDECR', 'BDEDO', 'BDEGT', 'BDEHN', 'BDENI', 'BDEPA', 'BDESV']
DB_USER = 'haproxy'
DB_PASS = str(os.getenv('QAPASS'))
DB_HOST = '159.203.104.215'
BACKUP_PATH = '/backup/'
DATE_FORMAT = '%Y-%m-%d'


def RemoveFile(filename):
    try:
        os.remove(filename)
        return
    except BaseException as ex:
        print(f'FILE REMOVE ERROR {ex}')


def OpenFTP(server='FACTOR'):
    try:
        '''
        if(server == 'FACTOR'):
            ftp = ftplib.FTP("165.227.114.190")
            ftp.login("factor", 'rh$2021')
            ftp.cwd('upload')
            return ftp
        else:
            ## ---- firewall ftp port -> 2023
            ftp = ftplib.FTP()
            ftp.connect("190.122.187.218", 2023)
            ftp.login("ming", 'dit2020 2020')
            return ftp
        '''
        ftp = ftplib.FTP("165.227.114.190")
        ftp.login("factor", 'rh$2021')
        ftp.cwd('upload')
        return ftp
    except ftplib.Error as er:
        print(f'FTP open error { er }')
    return

def DeleteFromSSHStorage(filename):
    try:
        # ftp = OpenFTP('FIREWALL')
        ftp = OpenFTP()
        ftp.delete(filename)
        ftp.close()
    except BaseException as e:
        print(f'FTP DELETE ERROR {e}')
    return

def lessDate(fecha,dias):
    return ( fecha - timedelta(days=dias)).strftime(DATE_FORMAT)

def DeleteFactorRH():
    try:
        ftp = OpenFTP()
        NOW = datetime.now()
        PREV = lessDate(NOW,3);
        to_delete_docs = 'factorh_grupodit-' + PREV
        # to_delete_docs = 'factormysql-'
        files = [doc for doc in ftp.nlst() if to_delete_docs in doc ]

        for doc in files:
            ftp.delete(doc)

        ftp.close()
    except Exception as error:
        print(f'FTP FACTOR DELETE ERROR {error}')


def PutOverSSH(path='/home/ming/CODE/PyKup/backup/BDEDO-20082021-1145.sql',remote='mingmecca.sql'):
    try:
        ftp = OpenFTP('FIREWALL')
        with open(path, "rb") as file:
            ftp.storbinary('STOR %s' % remote, file)
        ftp.close()
        RemoveFile(path)
    except BaseException as err:
        print(f"FTP CREATE ERROR {err}")
    return

def MongoBK():
    BACKUP_PATH = os.getcwd() + '/backup/mongo/'
    try:
        os.stat(BACKUP_PATH)
    except:
        os.mkdir(BACKUP_PATH)

    mongo_url = 'mongodb://QAadmin:QA20172017@165.227.177.95:27017/test001?authSource=admin&readPreference=primary&appname=MongoDB%20Compass&ssl=false'
    command = 'mongodump'+ ' -o ' + BACKUP_PATH + ' --uri ' + mongo_url

    with Popen(command, stdout=PIPE, stderr=None, shell=True) as process:
        output = process.communicate()[0].decode("utf-8")
        print(output)


    FILE_NAME = 'MONGO-' + datetime.now().strftime(DATE_FORMAT)
    FILE_DELETE_NAME = 'MONGO-' + lessDate(datetime.now(), 3) + ".zip"

    shutil.make_archive(FILE_NAME, 'zip', BACKUP_PATH )

    if( os.path.isdir( BACKUP_PATH )):
        shutil.rmtree(BACKUP_PATH)

    FILE_NAME = FILE_NAME + ".zip"
    PutOverSSH(FILE_NAME, FILE_NAME)
    DeleteFromSSHStorage(FILE_DELETE_NAME)



def MysqlBK():
    BACKUP_PATH = 'backup/mysql/'
    NOW = datetime.now()

    DATETIME =  NOW.strftime(DATE_FORMAT)
    TO_DELETE = ( NOW - timedelta(days=3) ).strftime(DATE_FORMAT)

    for DB_NAME in DB_LIST:
        # FILE = f'{DB_NAME}-' + DATETIME + '.sql'
        FILE = f'{DB_NAME}-' + DATETIME + '.sql'
        FILE_NAME = BACKUP_PATH + FILE

        try:
            os.stat(BACKUP_PATH)
        except:
            os.mkdir(BACKUP_PATH)

        params = ' --routines=true --triggers=true --quick --compress --skip-lock-tables --verbose '
        command = "mysqldump -h " + DB_HOST + " -u " + DB_USER + " -p" + DB_PASS + " " + DB_NAME + params + " > " + FILE_NAME

        with Popen(command, stdout=PIPE, stderr=None, shell=True) as process:
            output = process.communicate()[0].decode("utf-8")
            print(output)

        PutOverSSH(FILE_NAME, FILE)
        DeleteFromSSHStorage( f'{DB_NAME}-' + TO_DELETE + '.sql' )
    return

MongoBK()
DeleteFactorRH()
MysqlBK()