abdullah/adb-sync
1
0
Fork 0
mirror of https://github.com/google/adb-sync.git synced 2026-01-03 01:48:02 +00:00
Code Issues Releases Wiki Activity

adb-sync: Stop assuming the encoding of the input files matches the

Browse Source 
current locale.

Instead, treat all file names as binary, EVEN for output.
This commit is contained in:
Rudolf Polzer
2015-12-20 19:35:22 +01:00
parent 9b9a7041ff
commit d21617f70b
1 changed files with 91 additions and 66 deletions
Download Patch File Download Diff File Expand all files Collapse all files

157
adb-sync
View File

@@ -16,10 +16,10 @@
"""Sync files from/to an Android device.""" """Sync files from/to an Android device."""
from __future__ import print_function
from __future__ import unicode_literals from __future__ import unicode_literals
import argparse import argparse
import glob import glob
import locale
import os import os
import re import re
import stat import stat
@@ -28,6 +28,32 @@ import sys
import time import time
def _sprintf(s, *args):
# To be able to use string formatting, we first have to covert to
# unicode strings; however, we must do so in a way that preserves all
# bytes, and convert back at the end. An encoding that maps all byte
# values to different Unicode codepoints is cp437.
return (s.decode('cp437') % tuple([
(x.decode('cp437') if type(x) == bytes else x) for x in args
])).encode('cp437')
def _print(s, *args):
"""Writes a binary string to stdout.
Args:
s: The binary format string to write.
args: The args for the format string.
"""
if hasattr(sys.stdout, 'buffer'):
# Python 3.
sys.stdout.buffer.write(_sprintf(s, *args) + b'\n')
sys.stdout.buffer.flush()
else:
# Python 2.
sys.stdout.write(_sprintf(s, *args) + b'\n')
class AdbFileSystem(object): class AdbFileSystem(object):
"""Mimics os's file interface but uses the adb utility.""" """Mimics os's file interface but uses the adb utility."""
@@ -41,7 +67,7 @@ class AdbFileSystem(object):
# - st_mode (but only about S_ISDIR and S_ISREG properties) # - st_mode (but only about S_ISDIR and S_ISREG properties)
# Therefore, we only capture parts of 'ls -l' output that we actually use. # Therefore, we only capture parts of 'ls -l' output that we actually use.
# The other fields will be filled with dummy values. # The other fields will be filled with dummy values.
LS_TO_STAT_RE = re.compile(r'''^ LS_TO_STAT_RE = re.compile(br'''^
(?: (?:
(?P<S_IFREG> -) | (?P<S_IFREG> -) |
(?P<S_IFBLK> b) | (?P<S_IFBLK> b) |
@@ -87,7 +113,7 @@ class AdbFileSystem(object):
match = self.LS_TO_STAT_RE.match(line) match = self.LS_TO_STAT_RE.match(line)
if match is None: if match is None:
print('Warning: could not parse %r.' % line) _print(b'Warning: could not parse %r.', line)
raise OSError('Unparseable ls -al result.') raise OSError('Unparseable ls -al result.')
groups = match.groupdict() groups = match.groupdict()
@@ -173,8 +199,8 @@ class AdbFileSystem(object):
] ]
for test_string in test_strings: for test_string in test_strings:
good = False good = False
with self.Stdout(self.adb + [b'shell', b'date +%s' % with self.Stdout(self.adb + [b'shell', _sprintf(b'date +%s',
self.QuoteArgument(test_string)]) as stdout: self.QuoteArgument(test_string))]) as stdout:
for line in stdout: for line in stdout:
line = line.rstrip(b'\r\n') line = line.rstrip(b'\r\n')
if line == test_string: if line == test_string:
@@ -185,15 +211,15 @@ class AdbFileSystem(object):
def listdir(self, path): # os's name, so pylint: disable=g-bad-name def listdir(self, path): # os's name, so pylint: disable=g-bad-name
"""List the contents of a directory.""" """List the contents of a directory."""
with self.Stdout(self.adb + [b'shell', b'ls -a %s' % with self.Stdout(self.adb + [b'shell', _sprintf(b'ls -a %s',
self.QuoteArgument(path)]) as stdout: self.QuoteArgument(path))]) as stdout:
for line in stdout: for line in stdout:
yield line.rstrip(b'\r\n') yield line.rstrip(b'\r\n')
def CacheDirectoryLstat(self, path): def CacheDirectoryLstat(self, path):
"""Cache lstat for a directory.""" """Cache lstat for a directory."""
with self.Stdout(self.adb + [b'shell', b'ls -al %s' % with self.Stdout(self.adb + [b'shell', _sprintf(b'ls -al %s',
self.QuoteArgument(path + b'/')]) as stdout: self.QuoteArgument(path + b'/'))]) as stdout:
for line in stdout: for line in stdout:
line = line.rstrip(b'\r\n') line = line.rstrip(b'\r\n')
try: try:
@@ -201,8 +227,7 @@ class AdbFileSystem(object):
except OSError: except OSError:
continue continue
if filename is None: if filename is None:
print('Warning: could not cache %s' % _print(b'Warning: could not cache %s', line)
line.decode('utf-8', errors='replace'))
else: else:
self.stat_cache[path + b'/' + filename] = statdata self.stat_cache[path + b'/' + filename] = statdata
@@ -210,8 +235,8 @@ class AdbFileSystem(object):
"""Stat a file.""" """Stat a file."""
if path in self.stat_cache: if path in self.stat_cache:
return self.stat_cache[path] return self.stat_cache[path]
with self.Stdout(self.adb + [b'shell', b'ls -ald %s' % with self.Stdout(self.adb + [b'shell', _sprintf(b'ls -ald %s',
self.QuoteArgument(path)]) as stdout: self.QuoteArgument(path))]) as stdout:
for line in stdout: for line in stdout:
line = line.rstrip(b'\r\n') line = line.rstrip(b'\r\n')
statdata, filename = self.LsToStat(line) statdata, filename = self.LsToStat(line)
@@ -221,20 +246,20 @@ class AdbFileSystem(object):
def unlink(self, path): # os's name, so pylint: disable=g-bad-name def unlink(self, path): # os's name, so pylint: disable=g-bad-name
"""Delete a file.""" """Delete a file."""
if subprocess.call(self.adb + [b'shell', b'rm %s' % if subprocess.call(self.adb + [b'shell', _sprintf(b'rm %s',
self.QuoteArgument(path)]) != 0: self.QuoteArgument(path))]) != 0:
raise OSError('unlink failed') raise OSError('unlink failed')
def rmdir(self, path): # os's name, so pylint: disable=g-bad-name def rmdir(self, path): # os's name, so pylint: disable=g-bad-name
"""Delete a directory.""" """Delete a directory."""
if subprocess.call(self.adb + [b'shell', b'rmdir %s' % if subprocess.call(self.adb + [b'shell', _sprintf(b'rmdir %s',
self.QuoteArgument(path)]) != 0: self.QuoteArgument(path))]) != 0:
raise OSError('rmdir failed') raise OSError('rmdir failed')
def makedirs(self, path): # os's name, so pylint: disable=g-bad-name def makedirs(self, path): # os's name, so pylint: disable=g-bad-name
"""Create a directory.""" """Create a directory."""
if subprocess.call(self.adb + [b'shell', b'mkdir -p %s' % if subprocess.call(self.adb + [b'shell', _sprintf(b'mkdir -p %s',
self.QuoteArgument(path)]) != 0: self.QuoteArgument(path))]) != 0:
raise OSError('mkdir failed') raise OSError('mkdir failed')
def utime(self, path, times): def utime(self, path, times):
@@ -242,17 +267,18 @@ class AdbFileSystem(object):
"""Set the time of a file to a specified unix time.""" """Set the time of a file to a specified unix time."""
atime, mtime = times atime, mtime = times
timestr = time.strftime(b'%Y%m%d.%H%M%S', time.localtime(mtime)) timestr = time.strftime(b'%Y%m%d.%H%M%S', time.localtime(mtime))
if subprocess.call(self.adb + [b'shell', b'touch -mt %s %s' % if subprocess.call(self.adb + [b'shell', _sprintf(b'touch -mt %s %s',
(timestr, self.QuoteArgument(path))]) != 0: timestr, self.QuoteArgument(path))]) != 0:
raise OSError('touch failed') raise OSError('touch failed')
timestr = time.strftime(b'%Y%m%d.%H%M%S', time.localtime(atime)) timestr = time.strftime(b'%Y%m%d.%H%M%S', time.localtime(atime))
if subprocess.call(self.adb + [b'shell', b'touch -at %s %s' % if subprocess.call(self.adb + [b'shell',_sprintf( b'touch -at %s %s',
(timestr, self.QuoteArgument(path))]) != 0: timestr, self.QuoteArgument(path))]) != 0:
raise OSError('touch failed') raise OSError('touch failed')
def glob(self, path): def glob(self, path):
with self.Stdout(self.adb + [b'shell', b'for p in %s; do echo "$p"; done' % with self.Stdout(self.adb + [b'shell',
path]) as stdout: _sprintf(b'for p in %s; do echo "$p"; done',
path)]) as stdout:
for line in stdout: for line in stdout:
yield line.rstrip(b'\r\n') yield line.rstrip(b'\r\n')
@@ -293,8 +319,7 @@ def BuildFileList(fs, path, prefix=b''):
if hasattr(fs, 'CacheDirectoryLstat'): if hasattr(fs, 'CacheDirectoryLstat'):
fs.CacheDirectoryLstat(path) fs.CacheDirectoryLstat(path)
except OSError: except OSError:
print('Warning: could not cache lstat for %s' % _print(b'Warning: could not cache lstat for %s', path)
path.decode('utf-8', errors='replace'))
files.sort() files.sort()
for n in files: for n in files:
for t in BuildFileList(fs, path + b'/' + n, prefix + b'/' + n): for t in BuildFileList(fs, path + b'/' + n, prefix + b'/' + n):
@@ -302,7 +327,7 @@ def BuildFileList(fs, path, prefix=b''):
elif stat.S_ISREG(statresult.st_mode): elif stat.S_ISREG(statresult.st_mode):
yield prefix, statresult yield prefix, statresult
else: else:
print('Note: unsupported file: %s' % path.decode('utf-8', errors='replace')) _print(b'Note: unsupported file: %s', path)
def DiffLists(a, b): def DiffLists(a, b):
@@ -400,13 +425,13 @@ class FileSyncer(object):
def ScanAndDiff(self): def ScanAndDiff(self):
"""Scans the local and remote locations and identifies differences.""" """Scans the local and remote locations and identifies differences."""
print('Scanning and diffing...') _print(b'Scanning and diffing...')
locallist = BuildFileList(os, self.local) locallist = BuildFileList(os, self.local)
remotelist = BuildFileList(self.adb, self.remote) remotelist = BuildFileList(self.adb, self.remote)
self.local_only, self.both, self.remote_only = DiffLists(locallist, self.local_only, self.both, self.remote_only = DiffLists(locallist,
remotelist) remotelist)
if not self.local_only and not self.both and not self.remote_only: if not self.local_only and not self.both and not self.remote_only:
print('No files seen. User error?') _print(b'No files seen. User error?')
self.src_to_dst = (self.local_to_remote, self.remote_to_local) self.src_to_dst = (self.local_to_remote, self.remote_to_local)
self.dst_to_src = (self.remote_to_local, self.local_to_remote) self.dst_to_src = (self.remote_to_local, self.local_to_remote)
self.src_only = (self.local_only, self.remote_only) self.src_only = (self.local_only, self.remote_only)
@@ -414,7 +439,7 @@ class FileSyncer(object):
self.src = (self.local, self.remote) self.src = (self.local, self.remote)
self.dst = (self.remote, self.local) self.dst = (self.remote, self.local)
self.dst_fs = (self.adb, os) self.dst_fs = (self.adb, os)
self.push = ('Push', 'Pull') self.push = (b'Push', b'Pull')
self.copy = (self.adb.Push, self.adb.Pull) self.copy = (self.adb.Push, self.adb.Pull)
def InterruptProtection(self, fs, name): def InterruptProtection(self, fs, name):
@@ -443,9 +468,8 @@ class FileSyncer(object):
def __exit__(self, exc_type, exc_value, traceback): def __exit__(self, exc_type, exc_value, traceback):
if exc_type is not None: if exc_type is not None:
print('Interrupted-%s-Delete: %s' % _print(b'Interrupted-%s-Delete: %s' %
('Pull' if fs == os else 'Push', (b'Pull' if fs == os else b'Push', name))
name.decode('utf-8', errors='replace')))
if not dry_run: if not dry_run:
fs.unlink(name) fs.unlink(name)
return False return False
@@ -459,12 +483,12 @@ class FileSyncer(object):
for i in [0, 1]: for i in [0, 1]:
if self.src_to_dst[i] and not self.dst_to_src[i]: if self.src_to_dst[i] and not self.dst_to_src[i]:
if not self.src_only[i] and not self.both: if not self.src_only[i] and not self.both:
print('Cowardly refusing to delete everything.') _print(b'Cowardly refusing to delete everything.')
else: else:
for name, s in reversed(self.dst_only[i]): for name, s in reversed(self.dst_only[i]):
dst_name = self.dst[i] + name dst_name = self.dst[i] + name
print('%s-Delete: %s' % _print(b'%s-Delete: %s' %
(self.push[i], dst_name.decode('utf-8', errors='replace'))) (self.push[i], dst_name))
if stat.S_ISDIR(s.st_mode): if stat.S_ISDIR(s.st_mode):
if not self.dry_run: if not self.dry_run:
self.dst_fs[i].rmdir(dst_name) self.dst_fs[i].rmdir(dst_name)
@@ -499,7 +523,7 @@ class FileSyncer(object):
elif localminute < remoteminute: elif localminute < remoteminute:
l2r = False l2r = False
if l2r and r2l: if l2r and r2l:
print('Unresolvable: %s' % name.decode('utf-8', errors='replace')) _print(b'Unresolvable: %s', name)
continue continue
if l2r: if l2r:
i = 0 # Local to remote operation. i = 0 # Local to remote operation.
@@ -510,20 +534,21 @@ class FileSyncer(object):
src_stat = remotestat src_stat = remotestat
dst_stat = localstat dst_stat = localstat
dst_name = self.dst[i] + name dst_name = self.dst[i] + name
print('%s-Delete-Conflicting: %s' % _print(b'%s-Delete-Conflicting: %s', self.push[i], dst_name)
(self.push[i], dst_name.decode('utf-8', errors='replace')))
if stat.S_ISDIR(localstat.st_mode) or stat.S_ISDIR(remotestat.st_mode): if stat.S_ISDIR(localstat.st_mode) or stat.S_ISDIR(remotestat.st_mode):
if not self.allow_replace: if not self.allow_replace:
print('Would have to replace to do this. Use --force to allow this.') _print(b'Would have to replace to do this. '
b'Use --force to allow this.')
continue continue
if not self.allow_overwrite: if not self.allow_overwrite:
print('Would have to overwrite to do this, which --no-clobber forbids.') _print(b'Would have to overwrite to do this, '
b'which --no-clobber forbids.')
continue continue
if stat.S_ISDIR(dst_stat.st_mode): if stat.S_ISDIR(dst_stat.st_mode):
kill_files = [x for x in self.dst_only[i] kill_files = [x for x in self.dst_only[i]
if x[0][:len(name) + 1] == name + '/'] if x[0][:len(name) + 1] == name + b'/']
self.dst_only[i][:] = [x for x in self.dst_only[i] self.dst_only[i][:] = [x for x in self.dst_only[i]
if x[0][:len(name) + 1] != name + '/'] if x[0][:len(name) + 1] != name + b'/']
for l, s in reversed(kill_files): for l, s in reversed(kill_files):
if stat.S_ISDIR(s.st_mode): if stat.S_ISDIR(s.st_mode):
if not self.dry_run: if not self.dry_run:
@@ -550,8 +575,7 @@ class FileSyncer(object):
for name, s in self.src_only[i]: for name, s in self.src_only[i]:
src_name = self.src[i] + name src_name = self.src[i] + name
dst_name = self.dst[i] + name dst_name = self.dst[i] + name
print('%s: %s' % _print(b'%s: %s', self.push[i], dst_name)
(self.push[i], dst_name.decode('utf-8', errors='replace')))
if stat.S_ISDIR(s.st_mode): if stat.S_ISDIR(s.st_mode):
if not self.dry_run: if not self.dry_run:
self.dst_fs[i].makedirs(dst_name) self.dst_fs[i].makedirs(dst_name)
@@ -562,25 +586,25 @@ class FileSyncer(object):
self.num_bytes += s.st_size self.num_bytes += s.st_size
if not self.dry_run: if not self.dry_run:
if self.preserve_times: if self.preserve_times:
print('%s-Times: accessed %s, modified %s' % _print(b'%s-Times: accessed %s, modified %s' %
(self.push[i], (self.push[i],
time.asctime(time.localtime(s.st_atime)), time.asctime(time.localtime(s.st_atime)).encode('utf-8'),
time.asctime(time.localtime(s.st_mtime)))) time.asctime(time.localtime(s.st_mtime)).encode('utf-8')))
self.dst_fs[i].utime(dst_name, (s.st_atime, s.st_mtime)) self.dst_fs[i].utime(dst_name, (s.st_atime, s.st_mtime))
def TimeReport(self): def TimeReport(self):
"""Report time and amount of data transferred.""" """Report time and amount of data transferred."""
if self.dry_run: if self.dry_run:
print('Total: %d bytes' % self.num_bytes) _print(b'Total: %d bytes', self.num_bytes)
else: else:
end_time = time.time() end_time = time.time()
dt = end_time - self.start_time dt = end_time - self.start_time
rate = self.num_bytes / 1024.0 / dt rate = self.num_bytes / 1024.0 / dt
print('Total: %d KB/s (%d bytes in %.3fs)' % (rate, self.num_bytes, dt)) _print(b'Total: %d KB/s (%d bytes in %.3fs)', rate, self.num_bytes, dt)
def ExpandWildcards(globber, path): def ExpandWildcards(globber, path):
if path.find('?') == -1 and path.find('*') == -1 and path.find('[') == -1: if path.find(b'?') == -1 and path.find(b'*') == -1 and path.find(b'[') == -1:
return [path] return [path]
return globber.glob(path) return globber.glob(path)
@@ -667,20 +691,21 @@ def main(*args):
help='Do not do anything - just show what would '+ help='Do not do anything - just show what would '+
'be done.') 'be done.')
args = parser.parse_args() args = parser.parse_args()
args_encoding = locale.getdefaultlocale()[1]
localpatterns = [x.encode('utf-8') for x in args.source] localpatterns = [x.encode(args_encoding) for x in args.source]
remotepath = args.destination.encode('utf-8') remotepath = args.destination.encode(args_encoding)
adb = args.adb.encode('utf-8').split(b' ') adb = args.adb.encode(args_encoding).split(b' ')
if args.device: if args.device:
adb += [b'-d'] adb += [b'-d']
if args.emulator: if args.emulator:
adb += [b'-e'] adb += [b'-e']
if args.serial != None: if args.serial != None:
adb += [b'-s', args.serial.encode('utf-8')] adb += [b'-s', args.serial.encode(args_encoding)]
if args.host != None: if args.host != None:
adb += [b'-H', args.host.encode('utf-8')] adb += [b'-H', args.host.encode(args_encoding)]
if args.port != None: if args.port != None:
adb += [b'-P', args.port.encode('utf-8')] adb += [b'-P', args.port.encode(args_encoding)]
adb = AdbFileSystem(adb) adb = AdbFileSystem(adb)
# Expand wildcards. # Expand wildcards.
@@ -712,11 +737,11 @@ def main(*args):
local_to_remote, remote_to_local = remote_to_local, local_to_remote local_to_remote, remote_to_local = remote_to_local, local_to_remote
localpaths, remotepaths = remotepaths, localpaths localpaths, remotepaths = remotepaths, localpaths
if allow_replace and not allow_overwrite: if allow_replace and not allow_overwrite:
print('--no-clobber and --force are mutually exclusive.') _print(b'--no-clobber and --force are mutually exclusive.')
parser.print_help() parser.print_help()
return return
if delete_missing and local_to_remote and remote_to_local: if delete_missing and local_to_remote and remote_to_local:
print('--delete and --two-way are mutually exclusive.') _print(b'--delete and --two-way are mutually exclusive.')
parser.print_help() parser.print_help()
return return
@@ -724,19 +749,19 @@ def main(*args):
if (remote_to_local and local_to_remote) or delete_missing: if (remote_to_local and local_to_remote) or delete_missing:
if ((remote_to_local and len(localpaths) != len(set(localpaths))) or if ((remote_to_local and len(localpaths) != len(set(localpaths))) or
(local_to_remote and len(remotepaths) != len(set(remotepaths)))): (local_to_remote and len(remotepaths) != len(set(remotepaths)))):
print('--two-way and --delete are only supported for disjoint sets of '+ _print(b'--two-way and --delete are only supported for disjoint sets of '
'source and destination paths (in other words, all SRC must '+ b'source and destination paths (in other words, all SRC must '
'differ in basename).') b'differ in basename).')
parser.print_help() parser.print_help()
return return
for i in range(len(localpaths)): for i in range(len(localpaths)):
print('Sync: local %s, remote %s' % (localpaths[i], remotepaths[i])) _print(b'Sync: local %s, remote %s', localpaths[i], remotepaths[i])
syncer = FileSyncer(adb, localpaths[i], remotepaths[i], syncer = FileSyncer(adb, localpaths[i], remotepaths[i],
local_to_remote, remote_to_local, preserve_times, local_to_remote, remote_to_local, preserve_times,
delete_missing, allow_overwrite, allow_replace, dry_run) delete_missing, allow_overwrite, allow_replace, dry_run)
if not syncer.IsWorking(): if not syncer.IsWorking():
print('Device not connected or not working.') _print(b'Device not connected or not working.')
return return
try: try:
syncer.ScanAndDiff() syncer.ScanAndDiff()