1
# Copyright (C) 2011, 2012 Canonical Ltd
3
# This program is free software; you can redistribute it and/or modify
4
# it under the terms of the GNU General Public License as published by
5
# the Free Software Foundation; either version 2 of the License, or
6
# (at your option) any later version.
8
# This program is distributed in the hope that it will be useful,
9
# but WITHOUT ANY WARRANTY; without even the implied warranty of
10
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
11
# GNU General Public License for more details.
13
# You should have received a copy of the GNU General Public License
14
# along with this program; if not, write to the Free Software
15
# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
17
"""bzr-upload command implementations."""
19
from __future__ import absolute_import
29
lazy_import.lazy_import(globals(), """
43
auto_option = config.Option(
44
'upload_auto', default=False, from_unicode=config.bool_from_store,
46
Whether upload should occur when the tip of the branch changes.
48
auto_quiet_option = config.Option(
49
'upload_auto_quiet', default=False, from_unicode=config.bool_from_store,
51
Whether upload should occur quietly.
53
location_option = config.Option(
54
'upload_location', default=None,
56
The url to upload the working tree to.
58
revid_location_option = config.Option(
59
'upload_revid_location', default=u'.bzr-upload.revid',
61
The relative path to be used to store the uploaded revid.
63
The only bzr-related info uploaded with the working tree is the corresponding
64
revision id. The uploaded working tree is not linked to any other bzr data.
66
If the layout of your remote server is such that you can't write in the
67
root directory but only in the directories inside that root, you will need
68
to use the 'upload_revid_location' configuration variable to specify the
69
relative path to be used. That configuration variable can be specified in
70
locations.conf or branch.conf.
72
For example, given the following layout:
78
you may have write access in 'private' and 'public' but in 'Project'
79
itself. In that case, you can add the following in your locations.conf or
82
upload_revid_location = private/.bzr-upload.revid
86
# FIXME: Add more tests around invalid paths or relative paths that doesn't
87
# exist on remote (if only to get proper error messages) for
88
# 'upload_revid_location'
91
class BzrUploader(object):
93
def __init__(self, branch, to_transport, outf, tree, rev_id,
96
self.to_transport = to_transport
101
self._pending_deletions = []
102
self._pending_renames = []
103
self._uploaded_revid = None
106
def _up_stat(self, relpath):
107
return self.to_transport.stat(urlutils.escape(relpath))
109
def _up_rename(self, old_path, new_path):
110
return self.to_transport.rename(urlutils.escape(old_path),
111
urlutils.escape(new_path))
113
def _up_delete(self, relpath):
114
return self.to_transport.delete(urlutils.escape(relpath))
116
def _up_delete_tree(self, relpath):
117
return self.to_transport.delete_tree(urlutils.escape(relpath))
119
def _up_mkdir(self, relpath, mode):
120
return self.to_transport.mkdir(urlutils.escape(relpath), mode)
122
def _up_rmdir(self, relpath):
123
return self.to_transport.rmdir(urlutils.escape(relpath))
125
def _up_put_bytes(self, relpath, bytes, mode):
126
self.to_transport.put_bytes(urlutils.escape(relpath), bytes, mode)
128
def _up_get_bytes(self, relpath):
129
return self.to_transport.get_bytes(urlutils.escape(relpath))
131
def set_uploaded_revid(self, rev_id):
132
# XXX: Add tests for concurrent updates, etc.
133
revid_path = self.branch.get_config_stack().get('upload_revid_location')
134
self.to_transport.put_bytes(urlutils.escape(revid_path), rev_id)
135
self._uploaded_revid = rev_id
137
def get_uploaded_revid(self):
138
if self._uploaded_revid is None:
139
revid_path = self.branch.get_config_stack(
140
).get('upload_revid_location')
142
self._uploaded_revid = self._up_get_bytes(revid_path)
143
except errors.NoSuchFile:
144
# We have not uploaded to here.
145
self._uploaded_revid = revision.NULL_REVISION
146
return self._uploaded_revid
148
def _get_ignored(self):
149
if self._ignored is None:
151
ignore_file_path = '.bzrignore-upload'
152
ignore_file = self.tree.get_file(ignore_file_path)
153
except errors.NoSuchFile:
154
ignored_patterns = []
156
ignored_patterns = ignores.parse_ignore_file(ignore_file)
157
self._ignored = globbing.Globster(ignored_patterns)
160
def is_ignored(self, relpath):
161
glob = self._get_ignored()
162
ignored = glob.match(relpath)
165
# We still need to check that all parents are not ignored
166
dir = os.path.dirname(relpath)
167
while dir and not ignored:
168
ignored = glob.match(dir)
170
dir = os.path.dirname(dir)
173
def upload_file(self, old_relpath, new_relpath, mode=None):
175
if self.tree.is_executable(new_relpath):
180
self.outf.write('Uploading %s\n' % old_relpath)
182
old_relpath, self.tree.get_file_text(new_relpath), mode)
184
def _force_clear(self, relpath):
186
st = self._up_stat(relpath)
187
if stat.S_ISDIR(st.st_mode):
188
# A simple rmdir may not be enough
190
self.outf.write('Clearing %s/%s\n' % (
191
self.to_transport.external_url(), relpath))
192
self._up_delete_tree(relpath)
193
elif stat.S_ISLNK(st.st_mode):
195
self.outf.write('Clearing %s/%s\n' % (
196
self.to_transport.external_url(), relpath))
197
self._up_delete(relpath)
198
except errors.PathError:
201
def upload_file_robustly(self, relpath, mode=None):
202
"""Upload a file, clearing the way on the remote side.
204
When doing a full upload, it may happen that a directory exists where
205
we want to put our file.
207
self._force_clear(relpath)
208
self.upload_file(relpath, relpath, mode)
210
def upload_symlink(self, relpath, target):
211
self.to_transport.symlink(target, relpath)
213
def upload_symlink_robustly(self, relpath, target):
214
"""Handle uploading symlinks.
216
self._force_clear(relpath)
217
# Target might not be there at this time; dummy file should be
218
# overwritten at some point, possibly by another upload.
219
target = osutils.normpath(osutils.pathjoin(
220
osutils.dirname(relpath),
223
self.upload_symlink(relpath, target)
225
def make_remote_dir(self, relpath, mode=None):
228
self._up_mkdir(relpath, mode)
230
def make_remote_dir_robustly(self, relpath, mode=None):
231
"""Create a remote directory, clearing the way on the remote side.
233
When doing a full upload, it may happen that a file exists where we
234
want to create our directory.
237
st = self._up_stat(relpath)
238
if not stat.S_ISDIR(st.st_mode):
240
self.outf.write('Deleting %s/%s\n' % (
241
self.to_transport.external_url(), relpath))
242
self._up_delete(relpath)
244
# Ok the remote dir already exists, nothing to do
246
except errors.PathError:
248
self.make_remote_dir(relpath, mode)
250
def delete_remote_file(self, relpath):
252
self.outf.write('Deleting %s\n' % relpath)
253
self._up_delete(relpath)
255
def delete_remote_dir(self, relpath):
257
self.outf.write('Deleting %s\n' % relpath)
258
self._up_rmdir(relpath)
259
# XXX: Add a test where a subdir is ignored but we still want to
260
# delete the dir -- vila 100106
262
def delete_remote_dir_maybe(self, relpath):
263
"""Try to delete relpath, keeping failures to retry later."""
265
self._up_rmdir(relpath)
266
# any kind of PathError would be OK, though we normally expect
268
except errors.PathError:
269
self._pending_deletions.append(relpath)
271
def finish_deletions(self):
272
if self._pending_deletions:
273
# Process the previously failed deletions in reverse order to
274
# delete children before parents
275
for relpath in reversed(self._pending_deletions):
276
self._up_rmdir(relpath)
277
# The following shouldn't be needed since we use it once per
278
# upload, but better safe than sorry ;-)
279
self._pending_deletions = []
281
def rename_remote(self, old_relpath, new_relpath):
282
"""Rename a remote file or directory taking care of collisions.
284
To avoid collisions during bulk renames, each renamed target is
285
temporarily assigned a unique name. When all renames have been done,
286
each target get its proper name.
288
# We generate a sufficiently random name to *assume* that
289
# no collisions will occur and don't worry about it (nor
295
stamp = '.tmp.%.9f.%d.%d' % (time.time(),
297
random.randint(0, 0x7FFFFFFF))
299
self.outf.write('Renaming %s to %s\n' % (old_relpath, new_relpath))
300
self._up_rename(old_relpath, stamp)
301
self._pending_renames.append((stamp, new_relpath))
303
def finish_renames(self):
304
for (stamp, new_path) in self._pending_renames:
305
self._up_rename(stamp, new_path)
306
# The following shouldn't be needed since we use it once per upload,
307
# but better safe than sorry ;-)
308
self._pending_renames = []
310
def upload_full_tree(self):
311
self.to_transport.ensure_base() # XXX: Handle errors (add
312
# --create-prefix option ?)
313
with self.tree.lock_read():
314
for relpath, ie in self.tree.iter_entries_by_dir():
315
if relpath in ('', '.bzrignore', '.bzrignore-upload'):
317
# .bzrignore and .bzrignore-upload have no meaning outside
318
# a working tree so do not upload them
320
if self.is_ignored(relpath):
322
self.outf.write('Ignoring %s\n' % relpath)
324
if ie.kind == 'file':
325
self.upload_file_robustly(relpath)
326
elif ie.kind == 'symlink':
328
self.upload_symlink_robustly(
329
relpath, ie.symlink_target)
330
except errors.TransportNotPossible:
332
target = self.tree.path_content_summary(relpath)[3]
333
self.outf.write('Not uploading symlink %s -> %s\n'
335
elif ie.kind == 'directory':
336
self.make_remote_dir_robustly(relpath)
338
raise NotImplementedError
339
self.set_uploaded_revid(self.rev_id)
341
def upload_tree(self):
342
# If we can't find the revid file on the remote location, upload the
344
rev_id = self.get_uploaded_revid()
346
if rev_id == revision.NULL_REVISION:
348
self.outf.write('No uploaded revision id found,'
349
' switching to full upload\n')
350
self.upload_full_tree()
354
# Check if the revision hasn't already been uploaded
355
if rev_id == self.rev_id:
357
self.outf.write('Remote location already up to date\n')
359
from_tree = self.branch.repository.revision_tree(rev_id)
360
self.to_transport.ensure_base() # XXX: Handle errors (add
361
# --create-prefix option ?)
362
changes = self.tree.changes_from(from_tree)
363
with self.tree.lock_read():
364
for change in changes.removed:
365
if self.is_ignored(change.path[0]):
367
self.outf.write('Ignoring %s\n' % change.path[0])
369
if change.kind[0] == 'file':
370
self.delete_remote_file(change.path[0])
371
elif change.kind[0] == 'directory':
372
self.delete_remote_dir_maybe(change.path[0])
373
elif change.kind[0] == 'symlink':
374
self.delete_remote_file(change.path[0])
376
raise NotImplementedError
378
for change in changes.renamed:
379
if self.is_ignored(change.path[0]) and self.is_ignored(change.path[1]):
381
self.outf.write('Ignoring %s\n' % change.path[0])
382
self.outf.write('Ignoring %s\n' % change.path[1])
384
if change.changed_content:
385
# We update the change.path[0] content because renames and
386
# deletions are differed.
387
self.upload_file(change.path[0], change.path[1])
388
self.rename_remote(change.path[0], change.path[1])
389
self.finish_renames()
390
self.finish_deletions()
392
for change in changes.kind_changed:
393
if self.is_ignored(change.path[1]):
395
self.outf.write('Ignoring %s\n' % change.path[1])
397
if change.kind[0] in ('file', 'symlink'):
398
self.delete_remote_file(change.path[0])
399
elif change.kind[0] == 'directory':
400
self.delete_remote_dir(change.path[0])
402
raise NotImplementedError
404
if change.kind[1] == 'file':
405
self.upload_file(change.path[1], change.path[1])
406
elif change.kind[1] == 'symlink':
407
target = self.tree.get_symlink_target(change.path[1])
408
self.upload_symlink(change.path[1], target)
409
elif change.kind[1] == 'directory':
410
self.make_remote_dir(change.path[1])
412
raise NotImplementedError
414
for change in changes.added + changes.copied:
415
if self.is_ignored(change.path[1]):
417
self.outf.write('Ignoring %s\n' % change.path[1])
419
if change.kind[1] == 'file':
420
self.upload_file(change.path[1], change.path[1])
421
elif change.kind[1] == 'directory':
422
self.make_remote_dir(change.path[1])
423
elif change.kind[1] == 'symlink':
424
target = self.tree.get_symlink_target(change.path[1])
426
self.upload_symlink(change.path[1], target)
427
except errors.TransportNotPossible:
429
self.outf.write('Not uploading symlink %s -> %s\n'
430
% (change.path[1], target))
432
raise NotImplementedError
434
# XXX: Add a test for exec_change
435
for change in changes.modified:
436
if self.is_ignored(change.path[1]):
438
self.outf.write('Ignoring %s\n' % change.path[1])
440
if change.kind[1] == 'file':
441
self.upload_file(change.path[1], change.path[1])
442
elif change.kind[1] == 'symlink':
443
target = self.tree.get_symlink_target(change.path[1])
444
self.upload_symlink(change.path[1], target)
446
raise NotImplementedError
448
self.set_uploaded_revid(self.rev_id)
451
class CannotUploadToWorkingTree(errors.BzrCommandError):
453
_fmt = 'Cannot upload to a bzr managed working tree: %(url)s".'
456
class DivergedUploadedTree(errors.BzrCommandError):
458
_fmt = ("Your branch (%(revid)s)"
459
" and the uploaded tree (%(uploaded_revid)s) have diverged: ")
462
class cmd_upload(commands.Command):
463
"""Upload a working tree, as a whole or incrementally.
465
If no destination is specified use the last one used.
466
If no revision is specified upload the changes since the last upload.
468
Changes include files added, renamed, modified or removed.
470
_see_also = ['plugins/upload']
471
takes_args = ['location?']
476
option.Option('full', 'Upload the full working tree.'),
477
option.Option('quiet', 'Do not output what is being done.',
479
option.Option('directory',
480
help='Branch to upload from, '
481
'rather than the one containing the working directory.',
485
option.Option('auto',
486
'Trigger an upload from this branch whenever the tip '
490
def run(self, location=None, full=False, revision=None, remember=None,
491
directory=None, quiet=False, auto=None, overwrite=False
493
if directory is None:
497
relpath) = controldir.ControlDir.open_containing_tree_or_branch(
504
with locked.lock_read():
506
changes = wt.changes_from(wt.basis_tree())
508
if revision is None and changes.has_changed():
509
raise errors.UncommittedChanges(wt)
511
conf = branch.get_config_stack()
513
stored_loc = conf.get('upload_location')
514
if stored_loc is None:
515
raise errors.BzrCommandError(
516
'No upload location known or specified.')
518
# FIXME: Not currently tested
519
display_url = urlutils.unescape_for_display(stored_loc,
521
self.outf.write("Using saved location: %s\n" % display_url)
522
location = stored_loc
524
to_transport = transport.get_transport(location)
526
# Check that we are not uploading to a existing working tree.
528
to_bzr_dir = controldir.ControlDir.open_from_transport(
530
has_wt = to_bzr_dir.has_workingtree()
531
except errors.NotBranchError:
533
except errors.NotLocalUrl:
534
# The exception raised is a bit weird... but that's life.
538
raise CannotUploadToWorkingTree(url=location)
540
rev_id = branch.last_revision()
542
if len(revision) != 1:
543
raise errors.BzrCommandError(
544
'bzr upload --revision takes exactly 1 argument')
545
rev_id = revision[0].in_history(branch).rev_id
547
tree = branch.repository.revision_tree(rev_id)
549
uploader = BzrUploader(branch, to_transport, self.outf, tree,
553
prev_uploaded_rev_id = uploader.get_uploaded_revid()
554
graph = branch.repository.get_graph()
555
if not graph.is_ancestor(prev_uploaded_rev_id, rev_id):
556
raise DivergedUploadedTree(
557
revid=rev_id, uploaded_revid=prev_uploaded_rev_id)
559
uploader.upload_full_tree()
561
uploader.upload_tree()
563
# We uploaded successfully, remember it
564
with branch.lock_write():
565
upload_location = conf.get('upload_location')
566
if upload_location is None or remember:
567
conf.set('upload_location',
568
urlutils.unescape(to_transport.base))
570
conf.set('upload_auto', auto)
added
removed
breezy/plugins/upload/cmds.py
