Using Pexpect to Copy Resource Between Remote Machines

Using Pexpect to Copy Resource Between Remote Machines

A python script, use pexpect module to copy resource from machine B to machine C on machine A.

Examples:

SCP

python CopyResourceUtil.py --sm srcMachine --su david --sp david \

--sdirs '/home/david/test/abc.txt /home/david/test/123.txt' --dm destMachine --du bob --dp bob \

--destDirs /home/bob/test --destLoginPrompt '#' -p scp


FTP

python CopyResourceUtil.py --sm srcMachine --su david --sp david --sdirs '/home/david/music /home/david/books' \

--dm destMachine --du bob --dp bob --destDirs /home/bob/ --destLoginPrompt '#' -p ftp


Python 3 code:

#!/usr/bin/env python3
"""
Description:    
    This scripts is used to copy resource from source machine to destination machine using ftp or scp protocol between linux machines.
Usage:
    available option: --sm --srcMachine; --su --srcUser; --sp --srcPassword; --sdirs --srcDirs
                      --dm --destMachine; --du --destUser; --dp --destPassowrd;
                      --dru --destRootUser[Optional]; --drp --destRootPassword[Optional];
                      --ddirs --destDirs;  --destLoginPrompt[Optional]; --destRootPrompt[Optional]
                      -m --mode[Optional]; -p --protocol;
    -h,?,-?,--?,      print usage
    --h,--help            
    srcDirs and destDirs is delimited by space.
Example:
    python CopyResourceUtil.py --sm srcMachine --su srcUser --sp srcPassword --sdirs srcDirs \
    --dm destMachine --du destUser --dp destPassowrd --ddirs  destDirs -p ftp
        
    python CopyResourceUtil.py --sm srcMachine --su srcUser --sp srcPassword  --sdirs srcDirs \
    --dm destMachine --du destUser --dp destPassowrd --ddirs destDirs -p scp
"""
import pexpect
import os, sys,time, re, traceback
from optparse import OptionParser

def copy_resource_by_scp(srcMachine, srcUser, srcPassword, srcDirs,
                         destMachine, destUser, destPassowrd, destDirs,
                         destRootUser = None, destRootPassword = None, destLoginPrompt="~>", destRootPrompt="#", mode=None, logFile=None):
    """
    SSH to destination machine, and create destDirs,
    then on destination machine, use scp to copy resources from source machine to destination machine.
    
    srcDirs can be files or dirs
    destDirs should be dirs, if number of destDirs is less than number of srcDirs,
    redundant files/dirs in srcDirs would be copied to the last dir of destDirs
    """
    print ("Copy srcMachine=%s to destMachine=%s, destLoginPrompt=%s" % (srcMachine, destMachine, destLoginPrompt))
    changeToPrompt = '09876543211234567890'
    connected,dest_ssh_process =  ssh_login(ipAddress=destMachine, loginName=destUser, loginPasswd=destPassowrd, loginPrompt=destLoginPrompt,
              effectiveUser = destRootUser, effectivePassword = destRootPassword, effectivePrompt=destRootPrompt, changeToPrompt=changeToPrompt,logFile=logFile)
    if connected == False:
        print ("ERROR: Can't ssh to " + destMachine)
        return False
    destDirs = expand_destDirs(srcDirs, destDirs)

    for (source_dir, dest_dir) in (zip(srcDirs, destDirs)):
        #create dest_machine__dir if needed
        remoteCmd = "!ls " + dest_dir + " || " + "mkdir -p " + dest_dir
        print(remoteCmd)
        dest_ssh_process.sendline(remoteCmd)
        dest_ssh_process.expect(changeToPrompt)
        remoteCmd = 'scp -r ' + srcUser + '@' + srcMachine + ':' + source_dir + ' ' + dest_dir
        print(remoteCmd)
        dest_ssh_process.sendline(remoteCmd)
        while True:
            index = dest_ssh_process.expect(["assword", "(yes/no)",\
                                       pexpect.EOF, pexpect.TIMEOUT])
            if index == 0: # password
                break
            elif index == 1: # yes/no
                dest_ssh_process.sendline('yes')
            else:
                print("Failed to connect " + srcMachine)
                break          
        if index != 0:
            print("Copy resource failed, can not connect to " + srcMachine)
            break
        dest_ssh_process.sendline(srcPassword)
        index = dest_ssh_process.expect(['No such file or directory.*',"100%.*"])
        if index == 0:
            print('There is no ' + source_dir + ' on ' + srcMachine +', ignored!!!')
            continue
        elif index == 1:
            print("Successfully Copy %s to %s" % (source_dir, dest_dir))
        
        #change modes of downloaded files in dest machine
        if mode is not None:
            remoteCmd = 'chmod -R ' + str(mode) + ' ' + dest_dir
            print(remoteCmd)
            dest_ssh_process.sendline(remoteCmd)
            dest_ssh_process.expect(changeToPrompt)            
    #quit ssh session
    close_pexpect(dest_ssh_process)

def copy_resource_by_ftp(srcMachine, srcUser, srcPassword, srcDirs,
                         destMachine, destUser, destPassowrd, destDirs,
                         destLoginPrompt='#',ftpLoginPrompt='ftp>', mode=None, logFile=sys.stdout):
    """
    Telnet to destination machine, and create destDirs,
    then on destination machine, ftp to source machine and copy resources from source machine to destination machine.
    srcDirs can be dirs or files, destDirs must be dirs.
    """
    print("Copy destMachine=%s, destUser=%s to destPassowrd=%s, loginPrompt=%s" % (srcMachine, srcDirs, destMachine, destDirs))
    changeToPrompt = '09876543211234567890'
    connected,dest_telnet_process =  telnet_login(ipAddress=destMachine, loginName=destUser, loginPasswd=destPassowrd, loginPrompt=destLoginPrompt,
              changeToPrompt=changeToPrompt,logFile=logFile)
    if connected == False:
        print("ERROR: Can't telnet to " + dest_machine_address)
        return False
    destDirs = expand_destDirs(srcDirs, destDirs)
    #ftp to source machine and copy resources from source machine to destination machine.
    local_ftp_cmd = 'ftp -i ' + srcMachine
    print(local_ftp_cmd)
    dest_telnet_process.sendline(local_ftp_cmd)
    index = dest_telnet_process.expect(['.*Name.*',"Unknown host",pexpect.EOF, pexpect.TIMEOUT])
    if index != 0:
        print('ERROR !!! Failed to ftp to ' + srcMachine)
        print('Please make sure it is in good state')
        close_pexpect(dest_telnet_process)
        return -1
    dest_telnet_process.sendline(srcUser)
    dest_telnet_process.expect('.*Password:.*')
    dest_telnet_process.sendline(srcPassword)
    dest_telnet_process.expect([ftpLoginPrompt, pexpect.EOF, pexpect.TIMEOUT])
    if index != 0:
        print('ERROR !!! Failed to ftp to ' + srcMachine)
        print('Please make sure it is in good state')
        close_pexpect(dest_telnet_process)
        return -1
    # change to binary mode
    run_ftp_command(dest_telnet_process,"bin", ftpLoginPrompt)
    for (source_dir,dest_dir) in (zip(srcDirs,destDirs)):
        recursive_copy_dirs(dest_telnet_process,source_dir,dest_dir,ftpLoginPrompt,mode)
    # at last quit telnet session
    close_pexpect(dest_telnet_process)                

def recursive_copy_dirs(dest_ftp_process,source_dir,dest_dir,ftpLoginPrompt,mode):
    #create dirs on dest machine if not exist
    local_ftp_cmd = "!ls " + dest_dir + " || " + "mkdir -p " + dest_dir
    run_ftp_command(dest_ftp_process,local_ftp_cmd, ftpLoginPrompt)
    chmod_local_file(dest_ftp_process,dest_dir, ftpLoginPrompt,mode)
    
    #use lcd to change dest machine dir
    remote_ftp_cmd = 'lcd ' + dest_dir
    run_ftp_command(dest_ftp_process,remote_ftp_cmd, ftpLoginPrompt)

    #need to determine source_dir is a dir or a file.
    try:
        is_dir = is_directory(dest_ftp_process,source_dir,ftpLoginPrompt)
    except FileNotExistOnSourceMachineException:
        return False
   
    if is_dir == False:
        download_plain_file(dest_ftp_process,source_dir,dest_dir, ftpLoginPrompt,mode)
        return True
    #Otherwise copy all files under the dir, already cd to it
    remote_ftp_cmd = 'ls -al'
    run_ftp_command(dest_ftp_process,remote_ftp_cmd, ftpLoginPrompt)
    response = dest_ftp_process.before
    remote_files_list = response.split('\n')
    
    for line in remote_files_list:
        fileds = line.split()
        if len(fileds) >= 9:
            #ignore . and ..
            download_file = fileds[len(fileds)-1]
            if download_file in ['.','..']:
                continue
            download_file_full_path = source_dir + os.path.sep + download_file
            mod_flag = fileds[0]
            #the first bit means it is a dir or file
            is_dir = (mod_flag[0] == "d")
            if is_dir:
                #if download_file is a dir, then cd to it, and recursely cop its all files
                new_source_dir = download_file_full_path
                new_dest_dir = dest_dir + os.path.sep + download_file               
                recursive_copy_dirs(dest_ftp_process,new_source_dir,new_dest_dir,ftpLoginPrompt,mode)
                
                #after copy all files under this subdir, go up to parent dir to continue copy other files
                remote_ftp_cmd = 'cd ' + source_dir
                run_ftp_command(dest_ftp_process,remote_ftp_cmd, ftpLoginPrompt)
                remote_ftp_cmd = 'lcd ' + dest_dir
                run_ftp_command(dest_ftp_process,remote_ftp_cmd, ftpLoginPrompt)
            else:
                #copy the file
                download_plain_file(dest_ftp_process,download_file_full_path,dest_dir,ftpLoginPrompt,mode)

def ssh_login(ipAddress, loginName, loginPasswd, loginPrompt,
              effectiveUser = None, effectivePassword = None, effectivePrompt="#", changeToPrompt=None,logFile=None):
    """
    Connect to the host by SSH, return true if succeed, false otherwise
    """
    cmd = 'ssh ' + loginName + '@' + ipAddress
    child = pexpect.spawn(cmd)
    connected = False
    if logFile != None:
        child.logfile = logFile
    while True:
        index = child.expect(["(?i)password", "\(yes/no\)\?",pexpect.EOF, pexpect.TIMEOUT])
        if index == 0: # password
            break
        elif index == 1: # yes/no
            child.sendline('yes')
        else:
            print("Failed to login " + ipAddress)
            break
    if index != 0:
        print('ERROR !!! Failed to SSH to ' + ipAddress)
        close_pexpect(child)
        connected = False
    else:
        time.sleep(1)
        child.sendline(loginPasswd)
        tries = 0
        while True:
            index = child.expect([loginPrompt, \
            "host you are seated", "terminal type", "enter play", \
             pexpect.EOF, pexpect.TIMEOUT], timeout=900)
            if index == 0: # password
                break
            else:
                child.sendline("\n")
                tries = tries + 1
                if (tries > 4):
                    print("Failed to login " + ipAddress)
                    close_pexpect(child)
                    break
    if index != 0:
        print('ERROR !!! Failed to SSH to ' + ipAddress + \
        " wrong password, or connection to the host is refused.")
        close_pexpect(child)
        connected = False
    else:
        if (effectiveUser):suToEffectiveUser(child,effectiveUser,effectivePrompt)
        if (changeToPrompt): changePrompt(child,changeToPrompt)
        child.setwinsize(360, 360)
        connected = True
    return connected,child

def telnet_login(ipAddress, loginName, loginPasswd, loginPrompt,
              effectiveUser = None, effectivePassword = None, effectivePrompt="#", changeToPrompt=None,logFile=None):
    """
    Connect to the host by Telnet, return true if succeed, false otherwise
    """
    child = pexpect.spawn('telnet', [ipAddress])
    if logFile != None:
        child.logfile = logFile
    connected = False
    tries = 0
    while True:
        index = child.expect(['(?i)login: .*', \
                                   pexpect.EOF, pexpect.TIMEOUT])
        if index == 0: # login prompt
            break
        else:
            child.sendline('\n')
            tries = tries + 1
            if (tries > 4):
                print("Failed to login " + ipAddress)
                break
    if index != 0:
        print ('ERROR !!! Failed to telnet to ' + ipAddress)
        connected = False
    else:
        child.sendline(loginName)
        child.expect("(?i)password")
        time.sleep(1)
        child.sendline(loginPasswd)
        while True:
            index = child.expect([loginPrompt, \
            "host you are seated", "terminal type", "enter display", \
            pexpect.EOF, pexpect.TIMEOUT], timeout=900)
            if index == 0: # login prompt
                break
            else:
                child.sendline('\n')
                tries = tries + 1
                if (tries > 4):
                    print("Failed to login " + ipAddress)
                    close_pexpect(child)
                    break
        if index == 0:
            if (effectiveUser):suToEffectiveUser(child,effectiveUser,effectivePrompt)
            if (changeToPrompt): changePrompt(child,changeToPrompt)
            connected = True
    return connected,child

def suToEffectiveUser(child,effectiveUser,effectivePrompt):
    child.sendline("su " + effectiveUser)
    child.expect('(?i)password:.*')
    child.sendline(effectivePassword)
    child.expect(effectivePrompt)
    
def changePrompt(child,changeToPrompt):
    if (changeToPrompt):
        child.sendline("PS1=" + changeToPrompt)
        child.expect(changeToPrompt)

def close_pexpect(child):
    if child.isalive():
        child.close(force=True)
def is_directory(dest_ftp_process,source_dir,ftpLoginPrompt):
    #Determine source_dir is a dir or a plain file.
    is_dir = True
    remote_ftp_cmd = 'cd ' + source_dir
    run_ftp_command(dest_ftp_process,remote_ftp_cmd, ftpLoginPrompt)
    match  = re.search("(Not a directory)|(No such file or directory)", dest_ftp_process.before,re.IGNORECASE)
    if match:
        print("%s is a file, or doesn't exist." % source_dir)
        is_dir = False
        #if parent dir still not exist, then we can determine that this dir doesn't exist
        parent_dir = os.path.dirname(source_dir)
        remote_ftp_cmd = 'cd ' + parent_dir
        run_ftp_command(dest_ftp_process,remote_ftp_cmd, ftpLoginPrompt)
        match  = re.search("(Not a directory)|(No such file or directory)", dest_ftp_process.before,re.IGNORECASE)
        if match:
            print("%s doesn't exist." % parent_dir)
            raise FileNotExistOnSourceMachineException
        else:
            print("%s is a plain file." % source_dir)
    return is_dir
        
def download_plain_file(dest_ftp_process,dest_file_fullpath,dest_dir,ftpLoginPrompt,mode):
    print("Download from %s to %s" % (dest_file_fullpath, dest_dir))
    download_file = os.path.basename(dest_file_fullpath)
    remote_ftp_cmd = "get " + download_file
    run_ftp_command(dest_ftp_process,remote_ftp_cmd, ftpLoginPrompt)
    chmod_local_file(dest_ftp_process,dest_file_fullpath, ftpLoginPrompt,mode)
    
def chmod_local_file(dest_ftp_process,dest_file_fullpath, ftpLoginPrompt,mode):
    #change modes of downloaded file in dest machine
    if mode is not None:
        local_ftp_cmd = "!chmod " + str(mode) + " " + dest_file_fullpath
        run_ftp_command(dest_ftp_process,local_ftp_cmd, ftpLoginPrompt)

def run_ftp_command(dest_ftp_process,ftp_cmd, ftpLoginPrompt):
    print(ftp_cmd)
    dest_ftp_process.sendline(ftp_cmd)
    dest_ftp_process.expect(ftpLoginPrompt)    
    
def expand_destDirs(srcDirs, destDirs):
    srcDirs_len = len(srcDirs)
    destDirs_len = len(destDirs)
    if srcDirs_len > destDirs_len:
        destDirs.extend([destDirs[-1] ]* (srcDirs_len - destDirs_len))
    return destDirs

class FileNotExistOnSourceMachineException(Exception):pass

def exit_with_usage():
    print(globals()['__doc__'])
    os._exit(1)    
    
def test():
    usage = "usage: %prog [options] arg"
    parser = OptionParser(usage)
    parser.add_option("--sm", '--srcMachine', dest="srcMachine", help="Mandatory: srcMachine")
    parser.add_option("--su", '--srcUser', dest="srcUser", help="Mandatory: srcUser")
    parser.add_option("--sp", '--srcPassword', dest="srcPassword", help="Mandatory: srcPassword")
    parser.add_option("--sdirs", '--srcDirs', dest="srcDirs", help="Mandatory: srcDirs")
    
    parser.add_option("--dm", '--destMachine', dest="destMachine", help="Mandatory: destMachine")
    parser.add_option("--du", '--destUser', dest="destUser", help="Mandatory: destUser")
    parser.add_option("--dp", '--destPassowrd', dest="destPassowrd", help="Mandatory: destPassowrd")
    parser.add_option("--dru", '--destRootUser', dest="destRootUser", help="Optional: destRootUser, Only for SCP Protocol, [default: %default]")
    parser.add_option("--drp", '--destRootPassword', dest="destRootPassword", help="Optional: destRootPassword, Only for SCP Protocol")
    parser.add_option("--ddirs", '--destDirs', dest="destDirs", help="Mandatory: destDirs")
    parser.add_option("--destLoginPrompt", dest="destLoginPrompt", help="Optional: destLoginPrompt, [default: %default]")
    parser.add_option('--destRootPrompt', dest="destRootPrompt", help="Optional: destRootLoginPrompt, Only for SCP Protocol, [default: %default]")
    
    parser.add_option('-m', '--mode', dest="mode", help="Optional: mode, [default: %default]")
    parser.add_option("-p", "--protocol", dest="protocol", help="Mandatory: protocol")
    parser.set_defaults(destLoginPrompt="#")
    (options, args) = parser.parse_args()
    print (options, args)
    #separate dirs by white space
    options.srcDirs = options.srcDirs.split()
    options.destDirs = options.destDirs.split()
    if(None in [options.srcMachine, options.srcUser, options.srcPassword, options.srcDirs, options.destMachine, options.destUser, options.destPassowrd, options.destDirs, options.protocol ]):
        exit_with_usage()
        
    logFile=sys.stdout
    if options.protocol.upper() == 'SCP':
        copy_resource_by_scp(options.srcMachine, options.srcUser, options.srcPassword, options.srcDirs,
                         options.destMachine, options.destUser, options.destPassowrd, options.destDirs,
                         options.destRootUser, options.destRootPassword, options.destLoginPrompt, options.destRootPrompt, mode=options.mode, logFile=logFile)
    elif options.protocol.upper() == 'FTP':
        copy_resource_by_ftp(options.srcMachine, options.srcUser, options.srcPassword, options.srcDirs,
                         options.destMachine, options.destUser, options.destPassowrd, options.destDirs,
                         options.destLoginPrompt, 'ftp>', mode=options.mode,logFile=logFile)
    else:
        print('Unsupported protocol!!!')
        return -1
if __name__ == "__main__":
    try:
        test()
        os._exit(0)
    except Exception as e:
        print(str(e))
        traceback.print_exc()
        os._exit(-1)
Post a Comment

Labels

Java (159) Lucene-Solr (112) Interview (61) All (58) J2SE (53) Algorithm (45) Soft Skills (38) Eclipse (33) Code Example (31) Linux (25) JavaScript (23) Spring (22) Windows (22) Web Development (20) Tools (19) Nutch2 (18) Bugs (17) Debug (16) Defects (14) Text Mining (14) J2EE (13) Network (13) Troubleshooting (13) PowerShell (11) Chrome (9) Design (9) How to (9) Learning code (9) Performance (9) Problem Solving (9) UIMA (9) html (9) Http Client (8) Maven (8) Security (8) bat (8) blogger (8) Big Data (7) Continuous Integration (7) Google (7) Guava (7) JSON (7) Shell (7) ANT (6) Coding Skills (6) Database (6) Lesson Learned (6) Programmer Skills (6) Scala (6) Tips (6) css (6) Algorithm Series (5) Cache (5) Dynamic Languages (5) IDE (5) System Design (5) adsense (5) xml (5) AIX (4) Code Quality (4) GAE (4) Git (4) Good Programming Practices (4) Jackson (4) Memory Usage (4) Miscs (4) OpenNLP (4) Project Managment (4) Spark (4) Testing (4) ads (4) regular-expression (4) Android (3) Apache Spark (3) Become a Better You (3) Concurrency (3) Eclipse RCP (3) English (3) Happy Hacking (3) IBM (3) J2SE Knowledge Series (3) JAX-RS (3) Jetty (3) Restful Web Service (3) Script (3) regex (3) seo (3) .Net (2) Android Studio (2) Apache (2) Apache Procrun (2) Architecture (2) Batch (2) Bit Operation (2) Build (2) Building Scalable Web Sites (2) C# (2) C/C++ (2) CSV (2) Career (2) Cassandra (2) Distributed (2) Fiddler (2) Firefox (2) Google Drive (2) Gson (2) How to Interview (2) Html Parser (2) Http (2) Image Tools (2) JQuery (2) Jersey (2) LDAP (2) Life (2) Logging (2) Python (2) Software Issues (2) Storage (2) Text Search (2) xml parser (2) AOP (1) Application Design (1) AspectJ (1) Chrome DevTools (1) Cloud (1) Codility (1) Data Mining (1) Data Structure (1) ExceptionUtils (1) Exif (1) Feature Request (1) FindBugs (1) Greasemonkey (1) HTML5 (1) Httpd (1) I18N (1) IBM Java Thread Dump Analyzer (1) JDK Source Code (1) JDK8 (1) JMX (1) Lazy Developer (1) Mac (1) Machine Learning (1) Mobile (1) My Plan for 2010 (1) Netbeans (1) Notes (1) Operating System (1) Perl (1) Problems (1) Product Architecture (1) Programming Life (1) Quality (1) Redhat (1) Redis (1) Review (1) RxJava (1) Solutions logs (1) Team Management (1) Thread Dump Analyzer (1) Visualization (1) boilerpipe (1) htm (1) ongoing (1) procrun (1) rss (1)

Popular Posts