mailsys_server/auto_extract.py

import os
import sys
import glob
import time
import threading
import paramiko


hdd_root = '/hdd/*'

def cmd_run(index, shell_cmd):
    sh = shell_cmd[index]
    os.system(sh)

def main():
    surgery_list = glob.glob(hdd_root)

    for id in surgery_list:
        if 'lost+found' in id:
            surgery_list.remove(id)
        else:
            pass

    for id in surgery_list:
        
        file_list=glob.glob(id + '/*')
        for file in file_list:
            if 'log.txt' in file:
                f = open(file, 'rt')
                logs = f.readlines()
                f.close()                
                if len(logs) == 2:
                    thread_list = []
                    f = open(file, 'at')
                    r_time = time.ctime(time.time())
                    f.write('exectue is successfully starts at ' + str(r_time) + '\n')
                    f.close()
                    cmd_list = ['docker run --rm --name extract_2 --gpus '"device=1"' -v /dev:/dev -v /home/mc365/sources:/sources -v /hdd:/hdd ellishuntingmoon/mailsys:0.5 python3 /sources/extract_02.py ' + id + '/', 'echo ext_1 start']
                    for index in range(0, len(cmd_list)):
                        thread_list.append(threading.Thread(target=cmd_run,args=(index,cmd_list)))
                        thread_list[index].start()         
                    os.system('docker run --rm --name extract_1 --gpus '"device=0"' -v /dev:/dev -v /home/mc365/sources:/sources -v /hdd:/hdd ellishuntingmoon/mailsys:0.5 python3 /sources/extract_01.py ' + id + '/')       
                    f = open(file, 'at')
                    r_time = time.ctime(time.time())
                    f.write('exectue is successfully ends at ' + str(r_time) + '\n')
                    f.close()

                    #NOTE: Starting global coordinate
                    f = open(file, 'at')
                    r_time = time.ctime(time.time())
                    f.write('global coordinate successfully starts at ' + str(r_time) + '\n')
                    f.close()
                    #FIXME: change coordinate_g.py when IMU sensor works
                    os.system('docker run --rm --name coordinate --gpus '"device=1"' -v /dev:/dev -v /home/mc365/sources:/sources -v /hdd:/hdd ellishuntingmoon/mailsys:0.5 python3 /sources/coordinate_g.py ' + id + '/ >> /dev/null')
                    f = open(file, 'at')
                    r_time = time.ctime(time.time())
                    f.write('global coordinate successfully ends at ' + str(r_time) + '\n')
                    f.close()


                    #NOTE:
                    f = open(file, 'at')
                    r_time = time.ctime(time.time())
                    f.write('kpi successfully starts at ' + str(r_time) + '\n')
                    f.close()
                    #FIXME: change coordinate_g.py when IMU sensor works and get generic optimizer for DBSCAN
                    os.system('docker run --rm --name kpi --gpus '"device=1"' -v /dev:/dev -v /home/mc365/sources:/sources -v /hdd:/hdd ellishuntingmoon/mailsys:0.5 python3 /sources/kpi.py ' + id + '/ >> /dev/null')
                    f = open(file, 'at')
                    r_time = time.ctime(time.time())
                    f.write('kpi successfully ends at ' + str(r_time) + '\n')
                    f.close()

                    transport = paramiko.Transport(('365mc.iptime.org', 63122))
                    transport.connect(username='mc365', password='tkadbrdhMC1!')
                    sftp = paramiko.SFTPClient.from_transport(transport)
                    #FIXME: send file recursively
                    full_list = glob.glob(id + "/*")
                    for f in full_list:
                        print(f)
                        print(f[5:])
                        sftp.put(f,'/data/'+str(f[5:]))
                    sftp.close()
                    transport.close()      
                    os.system('rm -rf '+id)     

                else:
                    pass

if __name__ == '__main__':
    while True:
        main()
        time.sleep(900)