1
0
mirror of https://github.com/google/adb-sync.git synced 2026-01-03 01:48:02 +00:00
Files
adb-sync/adb-sync
2014-09-08 09:27:35 +02:00

688 lines
25 KiB
Python
Executable File

#!/usr/bin/python
# Copyright 2014 Google Inc. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Sync files from/to an Android device."""
from __future__ import print_function
from __future__ import unicode_literals
import argparse
import os
import re
import stat
import subprocess
import sys
import time
class AdbFileSystem(object):
"""Mimics os's file interface but uses the adb utility."""
def __init__(self, adb):
self.stat_cache = {}
self.adb = adb
# Regarding parsing stat results, we only care for the following fields:
# - st_size
# - st_mtime
# - st_mode (but only about S_ISDIR and S_ISREG properties)
# Therefore, we only capture parts of 'ls -l' output that we actually use.
# The other fields will be filled with dummy values.
LS_TO_STAT_RE = re.compile(r'''^
(?:
(?P<S_IFREG> -) |
(?P<S_IFBLK> b) |
(?P<S_IFCHR> c) |
(?P<S_IFDIR> d) |
(?P<S_IFLNK> l) |
(?P<S_IFIFO> p) |
(?P<S_IFSOCK> s))
[-r][-w][-xsS]
[-r][-w][-xsS]
[-r][-w][-xtT] # Mode string.
[ ]
[^ ]+ # User name/ID.
[ ]+
[^ ]+ # Group name/ID.
[ ]+
(?(S_IFBLK) [^ ]+[ ]+[^ ]+[ ]+) # Device numbers.
(?(S_IFCHR) [^ ]+[ ]+[^ ]+[ ]+) # Device numbers.
(?(S_IFREG)
(?P<st_size> [0-9]+) # Size.
[ ]+)
(?P<st_mtime>
[0-9]{4}-[0-9]{2}-[0-9]{2} # Date.
[ ]
[0-9]{2}:[0-9]{2}) # Time.
[ ]
# Don't capture filename for symlinks (ambiguous).
(?(S_IFLNK) .* | (?P<filename> .*))
$''', re.DOTALL | re.VERBOSE)
def LsToStat(self, line):
"""Convert a line from 'ls -l' output to a stat result.
Args:
line: Output line of 'ls -l' on Android.
Returns:
os.stat_result for the line.
Raises:
OSError: if the given string is not a 'ls -l' output line (but maybe an
error message instead).
"""
match = self.LS_TO_STAT_RE.match(line)
if match is None:
print('Warning: could not parse %r.' % line)
raise OSError('Unparseable ls -al result.')
groups = match.groupdict()
# Get the values we're interested in.
st_mode = ( # 0755
stat.S_IRWXU | stat.S_IRGRP | stat.S_IXGRP | stat.S_IROTH | stat.S_IXOTH)
if groups['S_IFREG']: st_mode |= stat.S_IFREG
if groups['S_IFBLK']: st_mode |= stat.S_IFBLK
if groups['S_IFCHR']: st_mode |= stat.S_IFCHR
if groups['S_IFDIR']: st_mode |= stat.S_IFDIR
if groups['S_IFIFO']: st_mode |= stat.S_IFIFO
if groups['S_IFLNK']: st_mode |= stat.S_IFLNK
if groups['S_IFSOCK']: st_mode |= stat.S_IFSOCK
st_size = groups['st_size']
if st_size is not None:
st_size = int(st_size)
st_mtime = time.mktime(time.strptime(match.group('st_mtime').decode('utf-8'),
'%Y-%m-%d %H:%M'))
# Fill the rest with dummy values.
st_ino = 1
st_rdev = 0
st_nlink = 1
st_uid = -2 # Nobody.
st_gid = -2 # Nobody.
st_atime = st_ctime = st_mtime
stbuf = os.stat_result((st_mode, st_ino, st_rdev, st_nlink, st_uid, st_gid,
st_size, st_atime, st_mtime, st_ctime))
filename = groups['filename']
return stbuf, filename
def Stdout(self, *popen_args):
"""Closes the process's stdout when done.
Usage:
with Stdout(...) as stdout:
DoSomething(stdout)
Args:
popen_args: Arguments for subprocess.Popen; stdout=PIPE is implicitly
added.
Returns:
An object for use by 'with'.
"""
class Stdout(object):
def __init__(self, popen):
self.popen = popen
def __enter__(self):
return self.popen.stdout
def __exit__(self, exc_type, exc_value, traceback):
self.popen.stdout.close()
if self.popen.wait() != 0:
raise OSError('Subprocess exited with nonzero status.')
return False
return Stdout(subprocess.Popen(*popen_args, stdout=subprocess.PIPE))
def QuoteArgument(self, arg):
# Quotes an argument for use by adb shell.
# Usually, arguments in 'adb shell' use are put in double quotes by adb,
# but not in any way escaped.
arg = arg.replace(b'\\', b'\\\\')
arg = arg.replace(b'"', b'\\"')
arg = arg.replace(b'$', b'\\$')
arg = arg.replace(b'`', b'\\`')
# Sometimes adb is evil and puts us NOT in double quotes.
# So here's a horrible hack to always force double quote mode.
# Thanks to autoconf guys for describing this method in portable shell.
# $PATH is assumed to always be set, even on Android.
arg = b'${PATH+"' + arg + b'"}'
return arg
def IsWorking(self):
"""Tests the adb connection."""
# This string should contain all possible evil, but no percent signs.
# Note this code uses 'date' and not 'echo', as date just calls strftime
# while echo does its own backslash escape handling additionally to the
# shell's. Too bad printf "%s\n" is not available.
test_strings = [
b'(',
b'(; #`ls`$PATH\'"(\\\\\\\\){};!\xc0\xaf\xff\xc2\xbf'
]
for test_string in test_strings:
good = False
with self.Stdout(self.adb + [b'shell', b'date',
b'+' + self.QuoteArgument(test_string)]) as stdout:
for line in stdout:
line = line.rstrip(b'\r\n')
if line == test_string:
good = True
if not good:
return False
return True
def listdir(self, path): # os's name, so pylint: disable=g-bad-name
"""List the contents of a directory."""
with self.Stdout(self.adb + [b'shell', b'ls', b'-a',
self.QuoteArgument(path)]) as stdout:
for line in stdout:
yield line.rstrip(b'\r\n')
def CacheDirectoryLstat(self, path):
"""Cache lstat for a directory."""
with self.Stdout(self.adb + [b'shell', b'ls', b'-al',
self.QuoteArgument(path + b'/')]) as stdout:
for line in stdout:
line = line.rstrip(b'\r\n')
try:
statdata, filename = self.LsToStat(line)
except OSError:
continue
if filename is None:
print('Warning: could not cache %s' %
line.decode('utf-8', errors='replace'))
else:
self.stat_cache[path + b'/' + filename] = statdata
def lstat(self, path): # os's name, so pylint: disable=g-bad-name
"""Stat a file."""
if path in self.stat_cache:
return self.stat_cache[path]
with self.Stdout(self.adb + [b'shell', b'ls', b'-ald',
self.QuoteArgument(path)]) as stdout:
for line in stdout:
line = line.rstrip(b'\r\n')
statdata, filename = self.LsToStat(line)
self.stat_cache[path] = statdata
return statdata
raise OSError('No such file or directory')
def unlink(self, path): # os's name, so pylint: disable=g-bad-name
"""Delete a file."""
if subprocess.call(self.adb + [b'shell', b'rm', path]) != 0:
raise OSError('unlink failed')
def rmdir(self, path): # os's name, so pylint: disable=g-bad-name
"""Delete a directory."""
if subprocess.call(self.adb + [b'shell', b'rmdir', path]) != 0:
raise OSError('rmdir failed')
def makedirs(self, path): # os's name, so pylint: disable=g-bad-name
"""Create a directory."""
if subprocess.call(self.adb + [b'shell', b'mkdir', b'-p', path]) != 0:
raise OSError('mkdir failed')
def utime(self, path, times):
# TODO(rpolzer): Find out why this does not work (returns status 255).
"""Set the time of a file to a specified unix time."""
atime, mtime = times
timestr = time.strftime(b'%Y%m%d.%H%M%S', time.localtime(mtime))
if subprocess.call(self.adb + [b'shell', b'touch', b'-mt',
timestr, path]) != 0:
raise OSError('touch failed')
timestr = time.strftime(b'%Y%m%d.%H%M%S', time.localtime(atime))
if subprocess.call(self.adb + [b'shell', b'touch', b'-at',
timestr, path]) != 0:
raise OSError('touch failed')
def Push(self, src, dst):
"""Push a file from the local file system to the Android device."""
if subprocess.call(self.adb + [b'push', src, dst]) != 0:
raise OSError('push failed')
def Pull(self, src, dst):
"""Pull a file from the Android device to the local file system."""
if subprocess.call(self.adb + [b'pull', src, dst]) != 0:
raise OSError('pull failed')
def BuildFileList(fs, path, prefix=b''):
"""Builds a file list.
Args:
fs: File system provider (can be os or AdbFileSystem()).
path: Initial path.
prefix: Path prefix for output file names.
Yields:
File names from path (prefixed by prefix).
Directories are yielded before their contents.
"""
try:
statresult = fs.lstat(path)
except OSError:
return
if stat.S_ISDIR(statresult.st_mode):
yield prefix, statresult
try:
files = list(fs.listdir(path))
except OSError:
return
try:
if hasattr(fs, 'CacheDirectoryLstat'):
fs.CacheDirectoryLstat(path)
except OSError:
print('Warning: could not cache lstat for %s' %
path.decode('utf-8', errors='replace'))
files.sort()
for n in files:
for t in BuildFileList(fs, path + b'/' + n, prefix + b'/' + n):
yield t
elif stat.S_ISREG(statresult.st_mode):
yield prefix, statresult
else:
print('Note: unsupported file: %s' % path.decode('utf-8', errors='replace'))
def DiffLists(a, b):
"""Compares two lists.
Args:
a: the first list.
b: the second list.
Returns:
a_only: the items from list a.
both: the items from both list, with the remaining tuple items combined.
b_only: the items from list b.
"""
a_only = []
b_only = []
both = []
a_iter = iter(a)
b_iter = iter(b)
a_active = True
b_active = True
a_available = False
b_available = False
a_item = None
b_item = None
while a_active and b_active:
if not a_available:
try:
a_item = next(a_iter)
a_available = True
except StopIteration:
a_active = False
break
if not b_available:
try:
b_item = next(b_iter)
b_available = True
except StopIteration:
b_active = False
break
if a_item[0] == b_item[0]:
both.append(tuple([a_item[0]] + list(a_item[1:]) + list(b_item[1:])))
a_available = False
b_available = False
elif a_item[0] < b_item[0]:
a_only.append(a_item)
a_available = False
elif a_item[0] > b_item[0]:
b_only.append(b_item)
b_available = False
else:
raise
if a_active:
if a_available:
a_only.append(a_item)
for item in a_iter:
a_only.append(item)
if b_active:
if b_available:
b_only.append(b_item)
for item in b_iter:
b_only.append(item)
return a_only, both, b_only
class FileSyncer(object):
"""File synchronizer."""
def __init__(self, adb, local_path, remote_path, local_to_remote,
remote_to_local, preserve_times, delete_missing, allow_overwrite,
allow_replace, dry_run):
self.local = local_path
self.remote = remote_path
self.adb = AdbFileSystem(adb)
self.local_to_remote = local_to_remote
self.remote_to_local = remote_to_local
self.preserve_times = preserve_times
self.delete_missing = delete_missing
self.allow_overwrite = allow_overwrite
self.allow_replace = allow_replace
self.dry_run = dry_run
self.local_only = None
self.both = None
self.remote_only = None
self.num_bytes = 0
self.start_time = time.time()
def IsWorking(self):
"""Tests the adb connection."""
return self.adb.IsWorking()
def ScanAndDiff(self):
"""Scans the local and remote locations and identifies differences."""
print('Scanning and diffing...')
locallist = BuildFileList(os, self.local)
remotelist = BuildFileList(self.adb, self.remote)
self.local_only, self.both, self.remote_only = DiffLists(locallist,
remotelist)
if not self.local_only and not self.both and not self.remote_only:
print('No files seen. User error?')
self.src_to_dst = (self.local_to_remote, self.remote_to_local)
self.dst_to_src = (self.remote_to_local, self.local_to_remote)
self.src_only = (self.local_only, self.remote_only)
self.dst_only = (self.remote_only, self.local_only)
self.src = (self.local, self.remote)
self.dst = (self.remote, self.local)
self.dst_fs = (self.adb, os)
self.push = ('Push', 'Pull')
self.copy = (self.adb.Push, self.adb.Pull)
def InterruptProtection(self, fs, name):
"""Sets up interrupt protection.
Usage:
with self.InterruptProtection(fs, name):
DoSomething()
If DoSomething() should get interrupted, the file 'name' will be deleted.
The exception otherwise will be passed on.
Args:
fs: File system object.
name: File name to delete.
Returns:
An object for use by 'with'.
"""
dry_run = self.dry_run
class DeleteInterruptedFile(object):
def __enter__(self):
pass
def __exit__(self, exc_type, exc_value, traceback):
if exc_type is not None:
print('Interrupted-%s-Delete: %s' %
('Pull' if fs == os else 'Push',
name.decode('utf-8', errors='replace')))
if not dry_run:
fs.unlink(name)
return False
return DeleteInterruptedFile()
def PerformDeletions(self):
"""Perform all deleting necessary for the file sync operation."""
if not self.delete_missing:
return
for i in [0, 1]:
if self.src_to_dst[i] and not self.dst_to_src[i]:
if not self.src_only[i] and not self.both:
print('Cowardly refusing to delete everything.')
else:
for name, s in reversed(self.dst_only[i]):
dst_name = self.dst[i] + name
print('%s-Delete: %s' %
(self.push[i], dst_name.decode('utf-8', errors='replace')))
if stat.S_ISDIR(s.st_mode):
if not self.dry_run:
self.dst_fs[i].rmdir(dst_name)
else:
if not self.dry_run:
self.dst_fs[i].unlink(dst_name)
del self.dst_only[i][:]
def PerformOverwrites(self):
"""Delete files/directories that are in the way for overwriting."""
src_only_prepend = ([], [])
for name, localstat, remotestat in self.both:
if stat.S_ISDIR(localstat.st_mode) and stat.S_ISDIR(remotestat.st_mode):
# A dir is a dir is a dir.
continue
elif stat.S_ISDIR(localstat.st_mode) or stat.S_ISDIR(remotestat.st_mode):
# Dir vs file? Nothing to do here yet.
pass
else:
# File vs file? Compare sizes.
if localstat.st_size == remotestat.st_size:
continue
l2r = self.local_to_remote
r2l = self.remote_to_local
if l2r and r2l:
# Truncate times to full minutes, as Android's "ls" only outputs minute
# accuracy.
localminute = int(localstat.st_mtime / 60)
remoteminute = int(remotestat.st_mtime / 60)
if localminute > remoteminute:
r2l = False
elif localminute < remoteminute:
l2r = False
if l2r and r2l:
print('Unresolvable: $%s' % name.decode('utf-8', errors='replace'))
continue
if l2r:
i = 0 # Local to remote operation.
src_stat = localstat
dst_stat = remotestat
else:
i = 1 # Remote to local operation.
src_stat = remotestat
dst_stat = localstat
dst_name = self.dst[i] + name
print('%s-Delete-Conflicting: %s' %
(self.push[i], dst_name.decode('utf-8', errors='replace')))
if stat.S_ISDIR(localstat.st_mode) or stat.S_ISDIR(remotestat.st_mode):
if not self.allow_replace:
print('Would have to replace to do this. Use --force to allow this.')
continue
if not self.allow_overwrite:
print('Would have to overwrite to do this, which --no-clobber forbids.')
continue
if stat.S_ISDIR(dst_stat.st_mode):
kill_files = [x for x in self.dst_only[i]
if x[0][:len(name) + 1] == name + '/']
self.dst_only[i][:] = [x for x in self.dst_only[i]
if x[0][:len(name) + 1] != name + '/']
for l, s in reversed(kill_files):
if stat.S_ISDIR(s.st_mode):
if not self.dry_run:
self.dst_fs[i].rmdir(self.dst[i] + l)
else:
if not self.dry_run:
self.dst_fs[i].unlink(self.dst[i] + l)
if not self.dry_run:
self.dst_fs[i].rmdir(dst_name)
elif stat.S_ISDIR(src_stat.st_mode):
if not self.dry_run:
self.dst_fs[i].unlink(dst_name)
else:
if not self.dry_run:
self.dst_fs[i].unlink(dst_name)
src_only_prepend[i].append((name, src_stat))
for i in [0, 1]:
self.src_only[i][:0] = src_only_prepend[i]
def PerformCopies(self):
"""Perform all copying necessary for the file sync operation."""
for i in [0, 1]:
if self.src_to_dst[i]:
for name, s in self.src_only[i]:
src_name = self.src[i] + name
dst_name = self.dst[i] + name
print('%s: %s' %
(self.push[i], dst_name.decode('utf-8', errors='replace')))
if stat.S_ISDIR(s.st_mode):
if not self.dry_run:
self.dst_fs[i].makedirs(dst_name)
else:
with self.InterruptProtection(self.dst_fs[i], dst_name):
if not self.dry_run:
self.copy[i](src_name, dst_name)
self.num_bytes += s.st_size
if not self.dry_run:
if self.preserve_times:
print('%s-Times: accessed %s, modified %s' %
(self.push[i],
time.asctime(time.localtime(s.st_atime)),
time.asctime(time.localtime(s.st_mtime))))
self.dst_fs[i].utime(dst_name, (s.st_atime, s.st_mtime))
def TimeReport(self):
"""Report time and amount of data transferred."""
if self.dry_run:
print('Total: %d bytes' % self.num_bytes)
else:
end_time = time.time()
dt = end_time - self.start_time
rate = self.num_bytes / 1024.0 / dt
print('Total: %d KB/s (%d bytes in %.3fs)' % (rate, self.num_bytes, dt))
def main(*args):
parser = argparse.ArgumentParser(
description='Synchronize a directory between an Android device and the '+
'local file system')
parser.add_argument('source', metavar='SRC', type=str,
help='The directory to read files/directories from. '+
'This must be a local path if -R is not specified, '+
'and an Android path if -R is specified.')
parser.add_argument('destination', metavar='DST', type=str,
help='The directory to write files/directories to. '+
'This must be an Android path if -R is not specified, '+
'and a local path if -R is specified.')
parser.add_argument('-e', '--adb', metavar='COMMAND', default='adb', type=str,
help='Use the given adb binary and arguments.')
parser.add_argument('--device', action='store_true',
help='Directs command to the only connected USB device; '+
'returns an error if more than one USB device is '+
'present. '+
'Corresponds to the "-d" option of adb.')
parser.add_argument('--emulator', action='store_true',
help='Directs command to the only running emulator; '+
'returns an error if more than one emulator is running. '+
'Corresponds to the "-e" option of adb.')
parser.add_argument('-s', '--serial', metavar='DEVICE', type=str,
help='Directs command to the device or emulator with '+
'the given serial number or qualifier. Overrides '+
'ANDROID_SERIAL environment variable. Use "adb devices" '+
'to list all connected devices with their respective '+
'serial number. '+
'Corresponds to the "-s" option of adb.')
parser.add_argument('-H', '--host', metavar='HOST', type=str,
help='Name of adb server host (default: localhost). '+
'Corresponds to the "-H" option of adb.')
parser.add_argument('-P', '--port', metavar='PORT', type=str,
help='Port of adb server (default: 5037). '+
'Corresponds to the "-P" option of adb.')
parser.add_argument('-R', '--reverse', action='store_true',
help='Reverse sync (pull, not push).')
parser.add_argument('-2', '--two-way', action='store_true',
help='Two-way sync (compare modification time; after '+
'the sync, both sides will have all files in the '+
'respective newest version. This relies on the clocks '+
'of your system and the device to match.')
#parser.add_argument('-t', '--times', action='store_true',
# help='Preserve modification times when copying.')
parser.add_argument('-d', '--delete', action='store_true',
help='Delete files from DST that are not present on '+
'SRC. Mutually exclusive with -2.')
parser.add_argument('-f', '--force', action='store_true',
help='Allow deleting files/directories when having to '+
'replace a file by a directory or vice versa. This is '+
'disabled by default to prevent large scale accidents.')
parser.add_argument('-n', '--no-clobber', action='store_true',
help='Do not ever overwrite any '+
'existing files. Mutually exclusive with -f.')
parser.add_argument('--dry-run',action='store_true',
help='Do not do anything - just show what would '+
'be done.')
args = parser.parse_args()
local = args.source
remote = args.destination
adb = args.adb.encode('utf-8').split(b' ')
if args.device:
adb += [b'-d']
if args.emulator:
adb += [b'-e']
if args.serial != None:
adb += [b'-s', args.serial.encode('utf-8')]
if args.host != None:
adb += [b'-H', args.host.encode('utf-8')]
if args.port != None:
adb += [b'-P', args.port.encode('utf-8')]
preserve_times = False # args.times
delete_missing = args.delete
allow_replace = args.force
allow_overwrite = not args.no_clobber
dry_run = args.dry_run
local_to_remote = True
remote_to_local = False
if args.two_way:
local_to_remote = True
remote_to_local = True
if args.reverse:
local_to_remote, remote_to_local = remote_to_local, local_to_remote
local, remote = remote, local
if allow_replace and not allow_overwrite:
print('--no-clobber and --force are mutually exclusive.')
parser.print_help()
return
if delete_missing and local_to_remote and remote_to_local:
print('--delete and --two-way are mutually exclusive.')
parser.print_help()
return
syncer = FileSyncer(adb,
local.encode('utf-8'), remote.encode('utf-8'),
local_to_remote, remote_to_local, preserve_times,
delete_missing, allow_overwrite, allow_replace, dry_run)
if not syncer.IsWorking():
print('Device not connected or not working.')
return
try:
syncer.ScanAndDiff()
syncer.PerformDeletions()
syncer.PerformOverwrites()
syncer.PerformCopies()
finally:
syncer.TimeReport()
if __name__ == '__main__':
main(*sys.argv)