136
148
if ie.revision is None:
137
149
# Need to store a new revision
138
150
ie.revision = revision_id
151
assert file_id is not None
139
152
assert ie.revision is not None
140
153
if ie.kind == 'symlink':
143
chunks = blob.chunked
144
texts.insert_record_stream([ChunkedContentFactory((file_id, ie.revision), tuple(parent_keys), ie.text_sha1, chunks)])
157
texts.insert_record_stream([FulltextContentFactory((file_id, ie.revision), tuple(parent_keys), ie.text_sha1, data)])
158
shamap = [(hexsha, "blob", (ie.file_id, ie.revision))]
146
if base_hexsha is not None:
147
old_path = path # Renames are not supported yet
148
if stat.S_ISDIR(base_mode):
149
invdelta.extend(remove_disappeared_children(base_inv, old_path,
150
lookup_object(base_hexsha), [], lookup_object))
162
if base_ie is not None:
163
old_path = base_inv.id2path(file_id)
164
if base_ie.kind == "directory":
165
invdelta.extend(remove_disappeared_children(old_path, base_ie.children, []))
153
168
invdelta.append((old_path, path, file_id, ie))
154
if base_hexsha != hexsha:
155
store_updater.add_object(blob, ie)
169
return (invdelta, shamap)
159
172
class SubmodulesRequireSubtrees(BzrError):
164
def import_git_submodule(texts, mapping, path, name, (base_hexsha, hexsha),
165
base_inv, parent_id, revision_id, parent_invs, lookup_object,
166
(base_mode, mode), store_updater):
167
if base_hexsha == hexsha and base_mode == mode:
177
def import_git_submodule(texts, mapping, path, hexsha, base_inv, base_ie,
178
parent_id, revision_id, parent_invs, shagitmap, lookup_object):
169
179
file_id = mapping.generate_file_id(path)
170
ie = TreeReference(file_id, name.decode("utf-8"), parent_id)
180
ie = TreeReference(file_id, urlutils.basename(path.decode("utf-8")),
171
182
ie.revision = revision_id
172
if base_hexsha is None:
187
if (base_ie.kind == ie.kind and
188
base_ie.reference_revision == ie.reference_revision):
189
ie.revision = base_ie.revision
176
190
ie.reference_revision = mapping.revision_id_foreign_to_bzr(hexsha)
177
texts.insert_record_stream([ChunkedContentFactory((file_id, ie.revision), (), None, [])])
191
texts.insert_record_stream([FulltextContentFactory((file_id, ie.revision), (), None, "")])
178
192
invdelta = [(oldpath, path, file_id, ie)]
182
def remove_disappeared_children(base_inv, path, base_tree, existing_children,
193
return invdelta, {}, {}
196
def remove_disappeared_children(path, base_children, existing_children):
185
for name, mode, hexsha in base_tree.iteritems():
186
if name in existing_children:
188
c_path = posixpath.join(path, name.decode("utf-8"))
189
ret.append((c_path, None, base_inv.path2id(c_path), None))
190
if stat.S_ISDIR(mode):
191
ret.extend(remove_disappeared_children(
192
base_inv, c_path, lookup_object(hexsha), [], lookup_object))
198
deletable = [(osutils.pathjoin(path, k), v) for k,v in base_children.iteritems() if k not in existing_children]
200
(path, ie) = deletable.pop()
201
ret.append((path, None, ie.file_id, None))
202
if ie.kind == "directory":
203
for name, child_ie in ie.children.iteritems():
204
deletable.append((osutils.pathjoin(path, name), child_ie))
196
def import_git_tree(texts, mapping, path, name, (base_hexsha, hexsha),
197
base_inv, parent_id, revision_id, parent_invs,
198
lookup_object, (base_mode, mode), store_updater, allow_submodules=False):
208
def import_git_tree(texts, mapping, path, hexsha, base_inv, base_ie, parent_id,
209
revision_id, parent_invs, shagitmap, lookup_object, allow_submodules=False):
199
210
"""Import a git tree object into a bzr repository.
201
212
:param texts: VersionedFiles object to add to
204
215
:param base_inv: Base inventory against which to return inventory delta
205
216
:return: Inventory delta for this subtree
207
if base_hexsha == hexsha and base_mode == mode:
208
# If nothing has changed since the base revision, we're done
211
219
file_id = mapping.generate_file_id(path)
212
220
# We just have to hope this is indeed utf-8:
213
ie = InventoryDirectory(file_id, name.decode("utf-8"), parent_id)
214
tree = lookup_object(hexsha)
215
if base_hexsha is None:
217
old_path = None # Newly appeared here
219
base_tree = lookup_object(base_hexsha)
220
old_path = path # Renames aren't supported yet
221
if base_tree is None or type(base_tree) is not Tree:
221
ie = InventoryDirectory(file_id, urlutils.basename(path.decode("utf-8")),
224
# Newly appeared here
222
225
ie.revision = revision_id
223
invdelta.append((old_path, path, ie.file_id, ie))
224
texts.insert_record_stream([ChunkedContentFactory((ie.file_id, ie.revision), (), None, [])])
226
texts.insert_record_stream([FulltextContentFactory((file_id, ie.revision), (), None, "")])
227
invdelta.append((None, path, file_id, ie))
229
# See if this has changed at all
231
base_sha = shagitmap.lookup_tree(file_id, base_inv.revision_id)
235
if base_sha == hexsha:
236
# If nothing has changed since the base revision, we're done
238
if base_ie.kind != "directory":
239
ie.revision = revision_id
240
texts.insert_record_stream([FulltextContentFactory((ie.file_id, ie.revision), (), None, "")])
241
invdelta.append((base_inv.id2path(ie.file_id), path, ie.file_id, ie))
242
if base_ie is not None and base_ie.kind == "directory":
243
base_children = base_ie.children
225
246
# Remember for next time
226
247
existing_children = set()
228
for child_mode, name, child_hexsha in tree.entries():
229
existing_children.add(name)
230
child_path = posixpath.join(path, name)
231
if type(base_tree) is Tree:
233
child_base_mode, child_base_hexsha = base_tree[name]
235
child_base_hexsha = None
238
child_base_hexsha = None
240
if stat.S_ISDIR(child_mode):
241
subinvdelta, grandchildmodes = import_git_tree(
242
texts, mapping, child_path, name,
243
(child_base_hexsha, child_hexsha),
244
base_inv, file_id, revision_id, parent_invs, lookup_object,
245
(child_base_mode, child_mode), store_updater,
250
tree = lookup_object(hexsha)
251
for mode, name, child_hexsha in tree.entries():
252
basename = name.decode("utf-8")
253
existing_children.add(basename)
254
child_path = osutils.pathjoin(path, name)
255
if stat.S_ISDIR(mode):
256
subinvdelta, grandchildmodes, subshamap = import_git_tree(
257
texts, mapping, child_path, child_hexsha, base_inv,
258
base_children.get(basename), file_id, revision_id,
259
parent_invs, shagitmap, lookup_object,
246
260
allow_submodules=allow_submodules)
247
elif S_ISGITLINK(child_mode): # submodule
261
invdelta.extend(subinvdelta)
262
child_modes.update(grandchildmodes)
263
shamap.extend(subshamap)
264
elif S_ISGITLINK(mode): # submodule
248
265
if not allow_submodules:
249
266
raise SubmodulesRequireSubtrees()
250
subinvdelta, grandchildmodes = import_git_submodule(
251
texts, mapping, child_path, name,
252
(child_base_hexsha, child_hexsha),
253
base_inv, file_id, revision_id, parent_invs, lookup_object,
254
(child_base_mode, child_mode), store_updater)
267
subinvdelta, grandchildmodes, subshamap = import_git_submodule(
268
texts, mapping, child_path, child_hexsha, base_inv, base_children.get(basename),
269
file_id, revision_id, parent_invs, shagitmap, lookup_object)
270
invdelta.extend(subinvdelta)
271
child_modes.update(grandchildmodes)
272
shamap.extend(subshamap)
256
subinvdelta = import_git_blob(texts, mapping,
257
child_path, name, (child_base_hexsha, child_hexsha),
258
base_inv, file_id, revision_id, parent_invs, lookup_object,
259
(child_base_mode, child_mode), store_updater)
261
child_modes.update(grandchildmodes)
262
invdelta.extend(subinvdelta)
263
if child_mode not in (stat.S_IFDIR, DEFAULT_FILE_MODE,
274
subinvdelta, subshamap = import_git_blob(texts, mapping,
275
child_path, child_hexsha, base_inv, base_children.get(basename), file_id,
276
revision_id, parent_invs, shagitmap, lookup_object,
277
mode_is_executable(mode), stat.S_ISLNK(mode))
278
invdelta.extend(subinvdelta)
279
shamap.extend(subshamap)
280
if mode not in (stat.S_IFDIR, DEFAULT_FILE_MODE,
264
281
stat.S_IFLNK, DEFAULT_FILE_MODE|0111):
265
child_modes[child_path] = child_mode
282
child_modes[child_path] = mode
266
283
# Remove any children that have disappeared
267
if base_tree is not None and type(base_tree) is Tree:
268
invdelta.extend(remove_disappeared_children(base_inv, old_path,
269
base_tree, existing_children, lookup_object))
270
store_updater.add_object(tree, ie)
271
return invdelta, child_modes
284
if base_ie is not None and base_ie.kind == "directory":
285
invdelta.extend(remove_disappeared_children(base_inv.id2path(file_id),
286
base_children, existing_children))
287
shamap.append((hexsha, "tree", (file_id, revision_id)))
288
return invdelta, child_modes, shamap
291
def approx_inv_size(inv):
292
# Very rough estimate, 1k per inventory entry
293
return len(inv) * 1024
274
296
def import_git_commit(repo, mapping, head, lookup_object,
278
300
# We have to do this here, since we have to walk the tree and
279
301
# we need to make sure to import the blobs / trees with the right
280
302
# path; this may involve adding them more than once.
281
parent_invs = parent_invs_cache.get_inventories(rev.parent_ids)
304
for parent_id in rev.parent_ids:
306
parent_invs.append(parent_invs_cache[parent_id])
308
parent_inv = repo.get_inventory(parent_id)
309
parent_invs.append(parent_inv)
310
parent_invs_cache[parent_id] = parent_inv
282
311
if parent_invs == []:
283
312
base_inv = Inventory(root_id=None)
287
315
base_inv = parent_invs[0]
288
base_tree = lookup_object(o.parents[0]).tree
289
base_mode = stat.S_IFDIR
290
store_updater = target_git_object_retriever._get_updater(rev)
291
store_updater.add_object(o, None)
292
inv_delta, unusual_modes = import_git_tree(repo.texts,
293
mapping, "", u"", (base_tree, o.tree), base_inv,
294
None, rev.revision_id, parent_invs, lookup_object,
295
(base_mode, stat.S_IFDIR), store_updater,
316
base_ie = base_inv.root
317
inv_delta, unusual_modes, shamap = import_git_tree(repo.texts,
318
mapping, "", o.tree, base_inv, base_ie, None, rev.revision_id,
319
parent_invs, target_git_object_retriever._idmap, lookup_object,
296
320
allow_submodules=getattr(repo._format, "supports_tree_reference", False))
297
store_updater.finish()
321
target_git_object_retriever._idmap.add_entries(shamap)
298
322
if unusual_modes != {}:
299
323
for path, mode in unusual_modes.iteritems():
300
324
warn_unusual_mode(rev.foreign_revid, path, mode)
369
397
# Order the revisions
370
398
# Create the inventory objects
372
400
revision_ids = topo_sort(graph)
374
402
if limit is not None:
375
403
revision_ids = revision_ids[:limit]
376
404
last_imported = None
377
405
for offset in range(0, len(revision_ids), batch_size):
378
target_git_object_retriever.start_write_group()
406
repo.start_write_group()
380
repo.start_write_group()
382
for i, head in enumerate(
383
revision_ids[offset:offset+batch_size]):
385
pb.update("fetching revisions", offset+i,
387
import_git_commit(repo, mapping, head, lookup_object,
388
target_git_object_retriever,
392
repo.abort_write_group()
395
hint = repo.commit_write_group()
397
pack_hints.extend(hint)
408
for i, head in enumerate(revision_ids[offset:offset+batch_size]):
410
pb.update("fetching revisions", offset+i, len(revision_ids))
411
import_git_commit(repo, mapping, head, lookup_object,
412
target_git_object_retriever,
399
target_git_object_retriever.abort_write_group()
416
repo.abort_write_group()
402
target_git_object_retriever.commit_write_group()
419
hint = repo.commit_write_group()
421
pack_hints.extend(hint)
422
target_git_object_retriever.commit_write_group()
403
423
return pack_hints, last_imported