1
# Copyright (C) 2011, 2012 Canonical Ltd
3
# This program is free software; you can redistribute it and/or modify
4
# it under the terms of the GNU General Public License as published by
5
# the Free Software Foundation; either version 2 of the License, or
6
# (at your option) any later version.
8
# This program is distributed in the hope that it will be useful,
9
# but WITHOUT ANY WARRANTY; without even the implied warranty of
10
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
11
# GNU General Public License for more details.
13
# You should have received a copy of the GNU General Public License
14
# along with this program; if not, write to the Free Software
15
# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
17
"""bzr-upload command implementations."""
19
from __future__ import absolute_import
28
lazy_import.lazy_import(globals(), """
47
auto_option = config.Option(
48
'upload_auto', default=False, from_unicode=config.bool_from_store,
50
Whether upload should occur when the tip of the branch changes.
52
auto_quiet_option = config.Option(
53
'upload_auto_quiet', default=False, from_unicode=config.bool_from_store,
55
Whether upload should occur quietly.
57
location_option = config.Option(
58
'upload_location', default=None,
60
The url to upload the working tree to.
62
revid_location_option = config.Option(
63
'upload_revid_location', default=u'.bzr-upload.revid',
65
The relative path to be used to store the uploaded revid.
67
The only bzr-related info uploaded with the working tree is the corresponding
68
revision id. The uploaded working tree is not linked to any other bzr data.
70
If the layout of your remote server is such that you can't write in the
71
root directory but only in the directories inside that root, you will need
72
to use the 'upload_revid_location' configuration variable to specify the
73
relative path to be used. That configuration variable can be specified in
74
locations.conf or branch.conf.
76
For example, given the following layout:
82
you may have write access in 'private' and 'public' but in 'Project'
83
itself. In that case, you can add the following in your locations.conf or
86
upload_revid_location = private/.bzr-upload.revid
90
# FIXME: Add more tests around invalid paths or relative paths that doesn't
91
# exist on remote (if only to get proper error messages) for
92
# 'upload_revid_location'
95
class BzrUploader(object):
97
def __init__(self, branch, to_transport, outf, tree, rev_id,
100
self.to_transport = to_transport
105
self._pending_deletions = []
106
self._pending_renames = []
107
self._uploaded_revid = None
110
def _up_stat(self, relpath):
111
return self.to_transport.stat(urlutils.escape(relpath))
113
def _up_rename(self, old_path, new_path):
114
return self.to_transport.rename(urlutils.escape(old_path),
115
urlutils.escape(new_path))
117
def _up_delete(self, relpath):
118
return self.to_transport.delete(urlutils.escape(relpath))
120
def _up_delete_tree(self, relpath):
121
return self.to_transport.delete_tree(urlutils.escape(relpath))
123
def _up_mkdir(self, relpath, mode):
124
return self.to_transport.mkdir(urlutils.escape(relpath), mode)
126
def _up_rmdir(self, relpath):
127
return self.to_transport.rmdir(urlutils.escape(relpath))
129
def _up_put_bytes(self, relpath, bytes, mode):
130
self.to_transport.put_bytes(urlutils.escape(relpath), bytes, mode)
132
def _up_get_bytes(self, relpath):
133
return self.to_transport.get_bytes(urlutils.escape(relpath))
135
def set_uploaded_revid(self, rev_id):
136
# XXX: Add tests for concurrent updates, etc.
137
revid_path = self.branch.get_config_stack().get('upload_revid_location')
138
self.to_transport.put_bytes(urlutils.escape(revid_path), rev_id)
139
self._uploaded_revid = rev_id
141
def get_uploaded_revid(self):
142
if self._uploaded_revid is None:
143
revid_path = self.branch.get_config_stack(
144
).get('upload_revid_location')
146
self._uploaded_revid = self._up_get_bytes(revid_path)
147
except errors.NoSuchFile:
148
# We have not uploaded to here.
149
self._uploaded_revid = revision.NULL_REVISION
150
return self._uploaded_revid
152
def _get_ignored(self):
153
if self._ignored is None:
155
ignore_file_path = '.bzrignore-upload'
156
ignore_file_id = self.tree.path2id(ignore_file_path)
157
ignore_file = self.tree.get_file(ignore_file_id,
159
ignored_patterns = ignores.parse_ignore_file(ignore_file)
160
except errors.NoSuchId:
161
ignored_patterns = []
162
self._ignored = globbing.Globster(ignored_patterns)
165
def is_ignored(self, relpath):
166
glob = self._get_ignored()
167
ignored = glob.match(relpath)
170
# We still need to check that all parents are not ignored
171
dir = os.path.dirname(relpath)
172
while dir and not ignored:
173
ignored = glob.match(dir)
175
dir = os.path.dirname(dir)
178
def upload_file(self, relpath, id, mode=None):
180
if self.tree.is_executable(id):
185
self.outf.write('Uploading %s\n' % relpath)
186
self._up_put_bytes(relpath, self.tree.get_file_text(id), mode)
188
def upload_file_robustly(self, relpath, id, mode=None):
189
"""Upload a file, clearing the way on the remote side.
191
When doing a full upload, it may happen that a directory exists where
192
we want to put our file.
195
st = self._up_stat(relpath)
196
if stat.S_ISDIR(st.st_mode):
197
# A simple rmdir may not be enough
199
self.outf.write('Clearing %s/%s\n' % (
200
self.to_transport.external_url(), relpath))
201
self._up_delete_tree(relpath)
202
except errors.PathError:
204
self.upload_file(relpath, id, mode)
206
def make_remote_dir(self, relpath, mode=None):
209
self._up_mkdir(relpath, mode)
211
def make_remote_dir_robustly(self, relpath, mode=None):
212
"""Create a remote directory, clearing the way on the remote side.
214
When doing a full upload, it may happen that a file exists where we
215
want to create our directory.
218
st = self._up_stat(relpath)
219
if not stat.S_ISDIR(st.st_mode):
221
self.outf.write('Deleting %s/%s\n' % (
222
self.to_transport.external_url(), relpath))
223
self._up_delete(relpath)
225
# Ok the remote dir already exists, nothing to do
227
except errors.PathError:
229
self.make_remote_dir(relpath, mode)
231
def delete_remote_file(self, relpath):
233
self.outf.write('Deleting %s\n' % relpath)
234
self._up_delete(relpath)
236
def delete_remote_dir(self, relpath):
238
self.outf.write('Deleting %s\n' % relpath)
239
self._up_rmdir(relpath)
240
# XXX: Add a test where a subdir is ignored but we still want to
241
# delete the dir -- vila 100106
243
def delete_remote_dir_maybe(self, relpath):
244
"""Try to delete relpath, keeping failures to retry later."""
246
self._up_rmdir(relpath)
247
# any kind of PathError would be OK, though we normally expect
249
except errors.PathError:
250
self._pending_deletions.append(relpath)
252
def finish_deletions(self):
253
if self._pending_deletions:
254
# Process the previously failed deletions in reverse order to
255
# delete children before parents
256
for relpath in reversed(self._pending_deletions):
257
self._up_rmdir(relpath)
258
# The following shouldn't be needed since we use it once per
259
# upload, but better safe than sorry ;-)
260
self._pending_deletions = []
262
def rename_remote(self, old_relpath, new_relpath):
263
"""Rename a remote file or directory taking care of collisions.
265
To avoid collisions during bulk renames, each renamed target is
266
temporarily assigned a unique name. When all renames have been done,
267
each target get its proper name.
269
# We generate a sufficiently random name to *assume* that
270
# no collisions will occur and don't worry about it (nor
276
stamp = '.tmp.%.9f.%d.%d' % (time.time(),
278
random.randint(0,0x7FFFFFFF))
280
self.outf.write('Renaming %s to %s\n' % (old_relpath, new_relpath))
281
self._up_rename(old_relpath, stamp)
282
self._pending_renames.append((stamp, new_relpath))
284
def finish_renames(self):
285
for (stamp, new_path) in self._pending_renames:
286
self._up_rename(stamp, new_path)
287
# The following shouldn't be needed since we use it once per upload,
288
# but better safe than sorry ;-)
289
self._pending_renames = []
291
def upload_full_tree(self):
292
self.to_transport.ensure_base() # XXX: Handle errors (add
293
# --create-prefix option ?)
294
self.tree.lock_read()
296
for relpath, ie in self.tree.iter_entries_by_dir():
297
if relpath in ('', '.bzrignore', '.bzrignore-upload'):
299
# .bzrignore and .bzrignore-upload have no meaning outside
300
# a working tree so do not upload them
302
if self.is_ignored(relpath):
304
self.outf.write('Ignoring %s\n' % relpath)
306
if ie.kind == 'file':
307
self.upload_file_robustly(relpath, ie.file_id)
308
elif ie.kind == 'directory':
309
self.make_remote_dir_robustly(relpath)
310
elif ie.kind == 'symlink':
312
target = self.tree.path_content_summary(relpath)[3]
313
self.outf.write('Not uploading symlink %s -> %s\n'
316
raise NotImplementedError
317
self.set_uploaded_revid(self.rev_id)
321
def upload_tree(self):
322
# If we can't find the revid file on the remote location, upload the
324
rev_id = self.get_uploaded_revid()
326
if rev_id == revision.NULL_REVISION:
328
self.outf.write('No uploaded revision id found,'
329
' switching to full upload\n')
330
self.upload_full_tree()
334
# Check if the revision hasn't already been uploaded
335
if rev_id == self.rev_id:
337
self.outf.write('Remote location already up to date\n')
339
from_tree = self.branch.repository.revision_tree(rev_id)
340
self.to_transport.ensure_base() # XXX: Handle errors (add
341
# --create-prefix option ?)
342
changes = self.tree.changes_from(from_tree)
343
self.tree.lock_read()
345
for (path, id, kind) in changes.removed:
346
if self.is_ignored(path):
348
self.outf.write('Ignoring %s\n' % path)
351
self.delete_remote_file(path)
352
elif kind is 'directory':
353
self.delete_remote_dir_maybe(path)
354
elif kind == 'symlink':
356
target = self.tree.path_content_summary(path)[3]
357
self.outf.write('Not deleting remote symlink %s -> %s\n'
360
raise NotImplementedError
362
for (old_path, new_path, id, kind,
363
content_change, exec_change) in changes.renamed:
364
if self.is_ignored(old_path) and self.is_ignored(new_path):
366
self.outf.write('Ignoring %s\n' % old_path)
367
self.outf.write('Ignoring %s\n' % new_path)
370
# We update the old_path content because renames and
371
# deletions are differed.
372
self.upload_file(old_path, id)
373
if kind == 'symlink':
375
self.outf.write('Not renaming remote symlink %s to %s\n'
376
% (old_path, new_path))
378
self.rename_remote(old_path, new_path)
379
self.finish_renames()
380
self.finish_deletions()
382
for (path, id, old_kind, new_kind) in changes.kind_changed:
383
if self.is_ignored(path):
385
self.outf.write('Ignoring %s\n' % path)
387
if old_kind == 'file':
388
self.delete_remote_file(path)
389
elif old_kind == 'directory':
390
self.delete_remote_dir(path)
392
raise NotImplementedError
394
if new_kind == 'file':
395
self.upload_file(path, id)
396
elif new_kind is 'directory':
397
self.make_remote_dir(path)
399
raise NotImplementedError
401
for (path, id, kind) in changes.added:
402
if self.is_ignored(path):
404
self.outf.write('Ignoring %s\n' % path)
407
self.upload_file(path, id)
408
elif kind == 'directory':
409
self.make_remote_dir(path)
410
elif kind == 'symlink':
412
target = self.tree.path_content_summary(path)[3]
413
self.outf.write('Not uploading symlink %s -> %s\n'
416
raise NotImplementedError
418
# XXX: Add a test for exec_change
420
content_change, exec_change) in changes.modified:
421
if self.is_ignored(path):
423
self.outf.write('Ignoring %s\n' % path)
426
self.upload_file(path, id)
428
raise NotImplementedError
430
self.set_uploaded_revid(self.rev_id)
435
class CannotUploadToWorkingTree(errors.BzrCommandError):
437
_fmt = 'Cannot upload to a bzr managed working tree: %(url)s".'
440
class DivergedUploadedTree(errors.BzrCommandError):
442
_fmt = ("Your branch (%(revid)s)"
443
" and the uploaded tree (%(uploaded_revid)s) have diverged: ")
446
class cmd_upload(commands.Command):
447
"""Upload a working tree, as a whole or incrementally.
449
If no destination is specified use the last one used.
450
If no revision is specified upload the changes since the last upload.
452
Changes include files added, renamed, modified or removed.
454
_see_also = ['plugins/upload']
455
takes_args = ['location?']
460
option.Option('full', 'Upload the full working tree.'),
461
option.Option('quiet', 'Do not output what is being done.',
463
option.Option('directory',
464
help='Branch to upload from, '
465
'rather than the one containing the working directory.',
469
option.Option('auto',
470
'Trigger an upload from this branch whenever the tip '
474
def run(self, location=None, full=False, revision=None, remember=None,
475
directory=None, quiet=False, auto=None, overwrite=False
477
if directory is None:
481
relpath) = bzrdir.BzrDir.open_containing_tree_or_branch(directory)
491
changes = wt.changes_from(wt.basis_tree())
493
if revision is None and changes.has_changed():
494
raise errors.UncommittedChanges(wt)
496
conf = branch.get_config_stack()
498
stored_loc = conf.get('upload_location')
499
if stored_loc is None:
500
raise errors.BzrCommandError(
501
'No upload location known or specified.')
503
# FIXME: Not currently tested
504
display_url = urlutils.unescape_for_display(stored_loc,
506
self.outf.write("Using saved location: %s\n" % display_url)
507
location = stored_loc
509
to_transport = transport.get_transport(location)
511
# Check that we are not uploading to a existing working tree.
513
to_bzr_dir = bzrdir.BzrDir.open_from_transport(to_transport)
514
has_wt = to_bzr_dir.has_workingtree()
515
except errors.NotBranchError:
517
except errors.NotLocalUrl:
518
# The exception raised is a bit weird... but that's life.
522
raise CannotUploadToWorkingTree(url=location)
524
rev_id = branch.last_revision()
526
if len(revision) != 1:
527
raise errors.BzrCommandError(
528
'bzr upload --revision takes exactly 1 argument')
529
rev_id = revision[0].in_history(branch).rev_id
531
tree = branch.repository.revision_tree(rev_id)
533
uploader = BzrUploader(branch, to_transport, self.outf, tree,
537
prev_uploaded_rev_id = uploader.get_uploaded_revid()
538
graph = branch.repository.get_graph()
539
if not graph.is_ancestor(prev_uploaded_rev_id, rev_id):
540
raise DivergedUploadedTree(
541
revid=rev_id, uploaded_revid=prev_uploaded_rev_id)
543
uploader.upload_full_tree()
545
uploader.upload_tree()
549
# We uploaded successfully, remember it
552
upload_location = conf.get('upload_location')
553
if upload_location is None or remember:
554
conf.set('upload_location',
555
urlutils.unescape(to_transport.base))
557
conf.set('upload_auto', auto)
added
removed
breezy/plugins/upload/cmds.py
