Use python's pexpect to copy files by ftp and scp

Use python's pexpect to copy files by ftp and scp

Following is the code recently I wrote to copy files by ftp and scp using python's pexpect module.

#!/usr/bin/env python

"""

Description:   

    This scripts(CopyResource.py) is used to copy resource from source machine to destination machine using ftp ro scp protocol between linux machines.

Usage:

    available option: --sm --srcMachine; --su --srcUser; --sp --srcPassword; --spath --srcPath

                      --dm --destMachine; --du --destUser; --dp --destPassowrd; --dpath --destPath;

                      -m --mode; -p --protocol;

    -h,?,-?,--?,      print usage

    --h,--help           

Example:

    python copyResource.py --sm srcMachine --su srcUser --sp srcPassword --spath srcPath \

    --dm destMachine --du destUser --dp destPassowrd --dpath  destPath -p ftp

       

    python copyResource.py --sm srcMachine --su srcUser --sp srcPassword  --spath srcPath \

    --dm destMachine --du destUser --dp destPassowrd --dpath destPath -p scp

"""

 

import pexpect

import os, getopt, sys

import traceback

 

COMMAND_TIMEOUT = 5000

SSH_KEYS = [ 'yes/no.*',  # 0, The first time login by ssh

         'Password:.*',  # 1, ssh: Password:

         'not known.*', # 2, ssh: Name or Service not known

         pexpect.EOF,

         pexpect.TIMEOUT

]

FTP_KEY = 'ftp>.*'

ROOT_PROMPT = '#'

PROMPTS_RE = '[#>]'

 

def copy_resource_by_scp(source_machine_address, source_machine_username, source_machine_password, source_machine__dirs,

                         dest_machine_address, dest_machine_username, dest_machine_password, dest_machine__dirs, mode = 777):

    """

    SSH to destination machine, and create dest_machine__dirs,

    then on destination machine, use scp to copy resources from source machine to destination machine.

    """   

    #ssh to destination machine

    cmd = 'ssh ' + dest_machine_username+ '@' + dest_machine_address

    print cmd

    dest_ssh_process = pexpect.spawn(cmd, timeout=COMMAND_TIMEOUT)

    index = dest_ssh_process.expect(SSH_KEYS, timeout=COMMAND_TIMEOUT) 

    if not index in [0,1]:

        print 'ERROR !!! Failed to connect to ' + dest_machine_address

        print 'Pls make sure it is in good state'

        if dest_ssh_process.isalive():

            dest_ssh_process.close(force=True)

            return -1

    if index == 0:

        dest_ssh_process.sendline('yes')

        index = dest_ssh_process.expect(SSH_KEYS)

    if index == 1:

        dest_ssh_process.sendline(source_machine_password) 

        dest_ssh_process.expect(PROMPTS_RE)

       

    for (source_dir,dest_dir) in (zip(source_machine__dirs,dest_machine__dirs)):

        #create dest_machine__dir

        cmd = 'mkdir -p ' + dest_dir

        print cmd

        dest_ssh_process.sendline(cmd)

        dest_ssh_process.expect(PROMPTS_RE)

       

        cmd = 'scp -r ' + source_machine_username + '@' + source_machine_address + ':' + source_dir + ' ' + dest_dir

        print  cmd

        dest_ssh_process.sendline(cmd)

        index = dest_ssh_process.expect(SSH_KEYS)      

        if index == 0:

            dest_ssh_process.sendline('yes')

            index = dest_ssh_process.expect(SSH_KEYS)       

        if index == 1:

            dest_ssh_process.sendline(source_machine_password)

            index = dest_ssh_process.expect(['No such file or directory.*',"100%.*"])

        if index == 0:

            print 'There is no ' + source_dir + ' on ' + source_machine_address +', ignored!!!'

            continue

                   

        #change modes of downloaded files in dest machine

        cmd = 'chmod -R ' + str(mode) +' ' + dest_dir

        print cmd

        dest_ssh_process.sendline(cmd)

        dest_ssh_process.expect(PROMPTS_RE)   

                 

    #quit ssh session

    cmd = 'exit'

    print cmd

    dest_ssh_process.sendline(cmd)

    if dest_ssh_process.isalive():

        dest_ssh_process.close(force=True)

 

def copy_resource_by_ftp(source_machine_address, source_machine_username, source_machine_password, source_machine__dirs,

                         dest_machine_address, dest_machine_username, dest_machine_password, dest_machine__dirs, mode = 777):

    """

    Telnet to destination machine, and create dest_machine__dirs,

    then on destination machine, ftp to source machine and copy resources from source machine to destination machine.

    """   

    #telnet to destination machine

    cmd = 'telnet ' + dest_machine_address

    print cmd

    dest_telnet_process = pexpect.spawn(cmd, timeout=COMMAND_TIMEOUT)

    index = dest_telnet_process.expect(['[l|L]ogin:', pexpect.EOF, pexpect.TIMEOUT])

 

    if index != 0:

        print 'ERROR !!! Failed to connect to ' + dest_machine_address

        print 'Pls make sure it is in good state'

        if dest_telnet_process.isalive():

            dest_telnet_process.close(force=True)

        os._exit(-1)   

   

    dest_telnet_process.sendline(dest_machine_username)

    print dest_machine_username

    dest_telnet_process.expect('.*Password:.*')

    dest_telnet_process.sendline(dest_machine_password)

    print dest_machine_password

    dest_telnet_process.expect(PROMPTS_RE)

    #user or password may be wrong

    print 'Have logined'

       

    print  'source_machine__dirs,dest_machine__dirs'

    print  source_machine__dirs,dest_machine__dirs

   

    #ftp to source machine and copy resources from source machine to destination machine.

    cmd = 'ftp ' + source_machine_address

    print cmd

    dest_telnet_process.sendline(cmd)

    dest_telnet_process.sendline(source_machine_username)

    dest_telnet_process.expect('.*Password:.*')

    dest_telnet_process.sendline(source_machine_password)

    dest_telnet_process.expect([FTP_KEY, pexpect.EOF, pexpect.TIMEOUT])

    if index != 0:

        print 'ERROR !!! Failed to connect to ' + source_machine_address

        print 'Pls make sure it is in good state'

        if dest_telnet_process.isalive():

            dest_telnet_process.close(force=True)

        os._exit(-2)      

   

    # change to binary mode

    dest_telnet_process.sendline('bin')   

    dest_telnet_process.expect(FTP_KEY)

    print 'bin'   

   

    for (source_dir,dest_dir) in (zip(source_machine__dirs,dest_machine__dirs)):

        #create dirs on dest machine, ftp doesn't support mkdir -p, so

        paths = dest_dir.split('/')

        mkdirPath = ''

        for path in paths:

            if path != '':

                mkdirPath += '/' + path

                cmd = 'mkdir ' + mkdirPath

                print cmd

                dest_telnet_process.sendline(cmd)

                dest_telnet_process.expect(FTP_KEY)                                                

       

        #in ftp, use lcd to change dest machine dir

        cmd = 'lcd ' + dest_dir

        print cmd

        dest_telnet_process.sendline(cmd)

        dest_telnet_process.expect(FTP_KEY)        

       

        #in ftp, change source machine's dir

        cmd = 'cd ' + source_dir 

        print cmd       

        dest_telnet_process.sendline (cmd)

        index = dest_telnet_process.expect(["No such file or directory.*", FTP_KEY]) 

        if index ==0:

            print 'failed to change dir to %s' % source_dir

            print 'There is no %s in %s' % (source_dir, source_machine_address)

            continue            

 

        #get all file names in source machine's dir

        cmd ='ls -al'

        print cmd

        dest_telnet_process.sendline(cmd)

        dest_telnet_process.expect(FTP_KEY)

    

        #the 9th column is the file name.

        cmd = "echo '" + dest_telnet_process.before + "' | awk '{ if (NF == 9) print $NF } '"

        print cmd

        fileNames = os.popen(cmd).read( ).splitlines();

        print 'fileNames'

        print fileNames

               

        #For ftp session,

        for fileName in fileNames:

            if fileName in ['.', '..']:

                continue

            cmd = 'get ' + fileName

            print cmd

            dest_telnet_process.sendline(cmd)

            dest_telnet_process.expect(FTP_KEY)

 

    # bye the ftp session after download all resources from source machine

    cmd = 'bye'

    print cmd

    dest_telnet_process.sendline(cmd)

    dest_telnet_process.expect(PROMPTS_RE)

   

    #change modes of downloaded files in dest machine

    for dest_dir in dest_machine__dirs:       

        cmd = 'chmod -R ' + str(mode) +' ' + dest_dir

        print cmd

        dest_telnet_process.sendline(cmd)

        dest_telnet_process.expect(PROMPTS_RE)   

   

    #quit telnet session

    cmd = 'exit'

    print cmd

    dest_telnet_process.sendline(cmd)   

    if dest_telnet_process.isalive():

        dest_telnet_process.close(force=True)

 

def exit_with_usage():

    print globals()['__doc__']

    os._exit(1)   

   

def test():

    optlist, args = getopt.getopt(sys.argv[1:], 'hm:p:?', ['sm=', 'su=','sp=', 'spath=','dm=', 'du=','dp=', 'dpath=','mode=', 'protocol='

                                                       'srcMachine=', 'srcUser=', 'srcPassword=', 'srcPath=',

                                                       'destMachine=', 'destUser=', 'destPassowrd=', 'destPath=',                                                      

                                                       "help",'h','?'])

    options = dict(optlist)

    print 'options'

    print options

    if [elem for elem in options if elem in ['-h','--h','?','-?','--?','--help']] or len(options)<9:

        exit_with_usage()

        

    srcMachine = None

    srcUser = None

    srcPassword = None

    srcPath = None

   

    destMachine = None

    destUser = None

    destPassowrd = None

    destPath = None

   

    mode = 777

    protocol = None

   

    for option, value in optlist:

        if option in ('--sm', '--srcMachine'):

            srcMachine = value

        elif option in ('--su','--srcUser'):

            srcUser = value

        elif option in ('--sp','--srcPassword'):

            srcPassword = value

        elif option in ('--spath','--srcPath'):

            srcPath = [value]

           

        if option in ('--dm', '--destMachine'):

            destMachine = value

        elif option in ('--du','--destUser'):

            destUser = value

        elif option in ('--dp','--destPassowrd'):

            destPassowrd = value

        elif option in ('--dpath','--destPath'):

            destPath = [value]

        elif option in ('-p','--protocol'):

            protocol = value.upper()

        elif option in ('-m','--mode'):

            mode = value

    print 'destPath'

    print destPath

    if srcMachine is None or srcUser is None or srcPassword is None or srcPath is None or \

        destMachine is None or destUser is None or destPassowrd is None or destPath is None or protocol is None:

         exit_with_usage()

        

    if protocol == 'SCP':

        copy_resource_by_scp(srcMachine, srcUser, srcPassword, srcPath,

                         destMachine, destUser, destPassowrd, destPath, mode)

    elif protocol == 'FTP':

        copy_resource_by_ftp(srcMachine, srcUser, srcPassword, srcPath,

                         destMachine, destUser, destPassowrd, destPath, mode)

    else:

        print 'Unsupported protocol!!!'

       

if __name__ == "__main__":

    try:       

        test()

        os._exit(0)

    except Exception, e:

        print str(e)

        traceback.print_exc()

        os._exit(1)


Post a Comment

Labels

Java (159) Lucene-Solr (110) All (60) Interview (59) J2SE (53) Algorithm (37) Eclipse (35) Soft Skills (35) Code Example (31) Linux (26) JavaScript (23) Spring (22) Windows (22) Web Development (20) Tools (19) Nutch2 (18) Bugs (17) Debug (15) Defects (14) Text Mining (14) J2EE (13) Network (13) PowerShell (11) Chrome (9) Continuous Integration (9) How to (9) Learning code (9) Performance (9) UIMA (9) html (9) Design (8) Dynamic Languages (8) Http Client (8) Maven (8) Security (8) Trouble Shooting (8) bat (8) blogger (8) Big Data (7) Google (7) Guava (7) JSON (7) Problem Solving (7) ANT (6) Coding Skills (6) Database (6) Scala (6) Shell (6) css (6) Algorithm Series (5) Cache (5) IDE (5) Lesson Learned (5) Miscs (5) Programmer Skills (5) System Design (5) Tips (5) adsense (5) xml (5) AIX (4) Code Quality (4) GAE (4) Git (4) Good Programming Practices (4) Jackson (4) Memory Usage (4) OpenNLP (4) Project Managment (4) Python (4) Spark (4) Testing (4) ads (4) regular-expression (4) Android (3) Apache Spark (3) Become a Better You (3) Concurrency (3) Eclipse RCP (3) English (3) Firefox (3) Happy Hacking (3) IBM (3) J2SE Knowledge Series (3) JAX-RS (3) Jetty (3) Restful Web Service (3) Script (3) regex (3) seo (3) .Net (2) Android Studio (2) Apache (2) Apache Procrun (2) Architecture (2) Batch (2) Build (2) Building Scalable Web Sites (2) C# (2) C/C++ (2) CSV (2) Career (2) Cassandra (2) Distributed (2) Fiddler (2) Google Drive (2) Gson (2) Html Parser (2) Http (2) Image Tools (2) JQuery (2) Jersey (2) LDAP (2) Life (2) Logging (2) Software Issues (2) Storage (2) Text Search (2) xml parser (2) AOP (1) Application Design (1) AspectJ (1) Bit Operation (1) Chrome DevTools (1) Cloud (1) Codility (1) Data Mining (1) Data Structure (1) ExceptionUtils (1) Exif (1) Feature Request (1) FindBugs (1) Greasemonkey (1) HTML5 (1) Httpd (1) I18N (1) IBM Java Thread Dump Analyzer (1) JDK Source Code (1) JDK8 (1) JMX (1) Lazy Developer (1) Mac (1) Machine Learning (1) Mobile (1) My Plan for 2010 (1) Netbeans (1) Notes (1) Operating System (1) Perl (1) Problems (1) Product Architecture (1) Programming Life (1) Quality (1) Redhat (1) Redis (1) Review (1) RxJava (1) Solutions logs (1) Team Management (1) Thread Dump Analyzer (1) Troubleshooting (1) Visualization (1) boilerpipe (1) htm (1) ongoing (1) procrun (1) rss (1)

Popular Posts