/brz/remove-bazaar : revision 0.200.814

1

2

#

3

# This program is free software; you can redistribute it and/or modify

4

# it under the terms of the GNU General Public License as published by

23

from dulwich.object_store import (

24

tree_lookup_path,

25

)

26

from itertools import (

27

imap,

28

)

29

import posixpath

30

26

import re

31

27

import stat

32

28

35

31

osutils,

36

32

trace,

37

33

ui,

34

urlutils,

38

35

)

39

36

from bzrlib.errors import (

40

37

BzrError,

53

50

from bzrlib.revision import (

54

51

NULL_REVISION,

55

52

)

56

from bzrlib.revisiontree import (

57

RevisionTree,

58

)

59

from bzrlib.testament import (

60

StrictTestament3,

61

)

62

53

from bzrlib.tsort import (

63

54

topo_sort,

64

55

)

68

59

69

60

from bzrlib.plugins.git.mapping import (

70

61

DEFAULT_FILE_MODE,

62

inventory_to_tree_and_blobs,

71

63

mode_is_executable,

72

mode_kind,

64

squash_revision,

73

65

warn_unusual_mode,

74

66

)

75

67

from bzrlib.plugins.git.object_store import (

76

68

BazaarObjectStore,

77

LRUTreeCache,

78

_tree_to_objects,

69

LRUInventoryCache,

79

70

)

80

71

from bzrlib.plugins.git.remote import (

81

72

RemoteGitRepository,

87

78

)

88

79

89

80

90

def import_git_blob(texts, mapping, path, name, (base_hexsha, hexsha),

91

base_inv, parent_id, revision_id,

92

parent_invs, lookup_object, (base_mode, mode), store_updater,

93

lookup_file_id):

81

def import_git_blob(texts, mapping, path, hexsha, base_hexsha,

82

base_inv, base_inv_shamap, base_ie, parent_id, revision_id,

83

parent_invs, lookup_object, executable, symlink):

94

84

"""Import a git blob object into a bzr repository.

95

85

96

86

:param texts: VersionedFiles to add to

98

88

:param blob: A git blob

99

89

:return: Inventory delta for this file

100

90

"""

101

if mapping.is_control_file(path):

102

return []

103

if base_hexsha == hexsha and base_mode == mode:

104

# If nothing has changed since the base revision, we're done

105

return []

106

file_id = lookup_file_id(path)

107

if stat.S_ISLNK(mode):

91

file_id = mapping.generate_file_id(path)

92

if symlink:

108

93

cls = InventoryLink

109

94

else:

110

95

cls = InventoryFile

111

ie = cls(file_id, name.decode("utf-8"), parent_id)

112

if ie.kind == "file":

113

ie.executable = mode_is_executable(mode)

114

if base_hexsha == hexsha and mode_kind(base_mode) == mode_kind(mode):

115

base_ie = base_inv[base_inv.path2id(path)]

96

# We just have to hope this is indeed utf-8:

97

ie = cls(file_id, urlutils.basename(path).decode("utf-8"), parent_id)

98

ie.executable = executable

99

# See if this has changed at all

100

if (base_hexsha == hexsha and base_ie.executable == ie.executable

101

and base_ie.kind == ie.kind):

102

# If nothing has changed since the base revision, we're done

103

return [], []

104

if base_hexsha == hexsha and base_ie.kind == ie.kind:

116

105

ie.text_size = base_ie.text_size

117

106

ie.text_sha1 = base_ie.text_sha1

118

if ie.kind == "symlink":

119

ie.symlink_target = base_ie.symlink_target

107

ie.symlink_target = base_ie.symlink_target

120

108

if ie.executable == base_ie.executable:

121

109

ie.revision = base_ie.revision

122

110

else:

126

114

if ie.kind == "symlink":

127

115

ie.revision = None

128

116

ie.symlink_target = blob.data

117

ie.text_size = None

118

ie.text_sha1 = None

129

119

else:

130

ie.text_size = sum(imap(len, blob.chunked))

131

ie.text_sha1 = osutils.sha_strings(blob.chunked)

120

ie.text_size = len(blob.data)

121

ie.text_sha1 = osutils.sha_string(blob.data)

132

122

# Check what revision we should store

133

123

parent_keys = []

134

124

for pinv in parent_invs:

135

try:

136

pie = pinv[file_id]

137

except NoSuchId:

138

continue

139

if (pie.text_sha1 == ie.text_sha1 and

140

pie.executable == ie.executable and

141

pie.symlink_target == ie.symlink_target):

125

if pinv.revision_id == base_inv.revision_id:

126

pie = base_ie

127

if pie is None:

128

continue

129

else:

130

try:

131

pie = pinv[file_id]

132

except NoSuchId:

133

continue

134

if pie.text_sha1 == ie.text_sha1 and pie.executable == ie.executable and pie.symlink_target == ie.symlink_target:

142

135

# found a revision in one of the parents to use

143

136

ie.revision = pie.revision

144

137

break

145

parent_key = (file_id, pie.revision)

146

if not parent_key in parent_keys:

147

parent_keys.append(parent_key)

138

parent_keys.append((file_id, pie.revision))

148

139

if ie.revision is None:

149

140

# Need to store a new revision

150

141

ie.revision = revision_id

142

assert file_id is not None

151

143

assert ie.revision is not None

152

144

if ie.kind == 'symlink':

153

145

chunks = []

154

146

else:

155

chunks = blob.chunked

156

texts.insert_record_stream([

157

ChunkedContentFactory((file_id, ie.revision),

158

tuple(parent_keys), ie.text_sha1, chunks)])

147

try:

148

chunks = blob.chunked

149

except AttributeError: # older version of dulwich

150

chunks = [blob.data]

151

texts.insert_record_stream([ChunkedContentFactory((file_id, ie.revision), tuple(parent_keys), ie.text_sha1, chunks)])

152

shamap = { ie.file_id: hexsha }

159

153

invdelta = []

160

if base_hexsha is not None:

161

old_path = path.decode("utf-8") # Renames are not supported yet

162

if stat.S_ISDIR(base_mode):

163

invdelta.extend(remove_disappeared_children(base_inv, old_path,

164

lookup_object(base_hexsha), [], lookup_object))

154

if base_ie is not None:

155

old_path = base_inv.id2path(file_id)

156

if base_ie.kind == "directory":

157

invdelta.extend(remove_disappeared_children(old_path, base_ie.children, []))

165

158

else:

166

159

old_path = None

167

new_path = path.decode("utf-8")

168

invdelta.append((old_path, new_path, file_id, ie))

169

if base_hexsha != hexsha:

170

store_updater.add_object(blob, ie, path)

171

return invdelta

160

invdelta.append((old_path, path, file_id, ie))

161

return (invdelta, shamap)

172

162

173

163

174

164

class SubmodulesRequireSubtrees(BzrError):

176

166

internal = False

177

167

178

168

179

def import_git_submodule(texts, mapping, path, name, (base_hexsha, hexsha),

180

base_inv, parent_id, revision_id, parent_invs, lookup_object,

181

(base_mode, mode), store_updater, lookup_file_id):

182

if base_hexsha == hexsha and base_mode == mode:

183

return [], {}

184

file_id = lookup_file_id(path)

185

ie = TreeReference(file_id, name.decode("utf-8"), parent_id)

169

def import_git_submodule(texts, mapping, path, hexsha, base_hexsha,

170

base_inv, base_ie, parent_id, revision_id, parent_invs, lookup_object):

171

file_id = mapping.generate_file_id(path)

172

ie = TreeReference(file_id, urlutils.basename(path.decode("utf-8")),

173

parent_id)

186

174

ie.revision = revision_id

187

if base_hexsha is None:

175

if base_ie is None:

188

176

oldpath = None

189

177

else:

190

178

oldpath = path

179

if (base_ie.kind == ie.kind and

180

base_ie.reference_revision == ie.reference_revision):

181

ie.revision = base_ie.revision

191

182

ie.reference_revision = mapping.revision_id_foreign_to_bzr(hexsha)

192

texts.insert_record_stream([

193

ChunkedContentFactory((file_id, ie.revision), (), None, [])])

183

texts.insert_record_stream([ChunkedContentFactory((file_id, ie.revision), (), None, [])])

194

184

invdelta = [(oldpath, path, file_id, ie)]

195

return invdelta, {}

196

197

198

def remove_disappeared_children(base_inv, path, base_tree, existing_children,

199

lookup_object):

200

"""Generate an inventory delta for removed children.

201

202

:param base_inv: Base inventory against which to generate the

203

inventory delta.

204

:param path: Path to process (unicode)

205

:param base_tree: Git Tree base object

206

:param existing_children: Children that still exist

207

:param lookup_object: Lookup a git object by its SHA1

208

:return: Inventory delta, as list

209

"""

210

assert type(path) is unicode

185

return invdelta, {}, {}

186

187

188

def remove_disappeared_children(path, base_children, existing_children):

211

189

ret = []

212

for name, mode, hexsha in base_tree.iteritems():

213

if name in existing_children:

214

continue

215

c_path = posixpath.join(path, name.decode("utf-8"))

216

file_id = base_inv.path2id(c_path)

217

assert file_id is not None

218

ret.append((c_path, None, file_id, None))

219

if stat.S_ISDIR(mode):

220

ret.extend(remove_disappeared_children(

221

base_inv, c_path, lookup_object(hexsha), [], lookup_object))

190

deletable = [(osutils.pathjoin(path, k), v) for k,v in base_children.iteritems() if k not in existing_children]

191

while deletable:

192

(path, ie) = deletable.pop()

193

ret.append((path, None, ie.file_id, None))

194

if ie.kind == "directory":

195

for name, child_ie in ie.children.iteritems():

196

deletable.append((osutils.pathjoin(path, name), child_ie))

222

197

return ret

223

198

224

199

225

def import_git_tree(texts, mapping, path, name, (base_hexsha, hexsha),

226

base_inv, parent_id, revision_id, parent_invs,

227

lookup_object, (base_mode, mode), store_updater,

228

lookup_file_id, allow_submodules=False):

200

def import_git_tree(texts, mapping, path, hexsha, base_hexsha, base_inv,

201

base_inv_shamap, base_ie, parent_id, revision_id, parent_invs,

202

lookup_object, allow_submodules=False):

229

203

"""Import a git tree object into a bzr repository.

230

204

231

205

:param texts: VersionedFiles object to add to

232

:param path: Path in the tree (str)

233

:param name: Name of the tree (str)

206

:param path: Path in the tree

234

207

:param tree: A git tree object

235

208

:param base_inv: Base inventory against which to return inventory delta

236

209

:return: Inventory delta for this subtree

237

210

"""

238

assert type(path) is str

239

assert type(name) is str

240

if base_hexsha == hexsha and base_mode == mode:

241

# If nothing has changed since the base revision, we're done

242

return [], {}

243

211

invdelta = []

244

file_id = lookup_file_id(path)

212

file_id = mapping.generate_file_id(path)

245

213

# We just have to hope this is indeed utf-8:

246

ie = InventoryDirectory(file_id, name.decode("utf-8"), parent_id)

214

ie = InventoryDirectory(file_id, urlutils.basename(path.decode("utf-8")),

215

parent_id)

216

if base_ie is None:

217

# Newly appeared here

218

ie.revision = revision_id

219

texts.insert_record_stream([ChunkedContentFactory((file_id, ie.revision), (), None, [])])

220

invdelta.append((None, path, file_id, ie))

221

else:

222

if base_hexsha == hexsha:

223

# If nothing has changed since the base revision, we're done

224

return [], {}, []

225

if base_ie.kind != "directory":

226

ie.revision = revision_id

227

texts.insert_record_stream([ChunkedContentFactory((ie.file_id, ie.revision), (), None, [])])

228

invdelta.append((base_inv.id2path(ie.file_id), path, ie.file_id, ie))

247

229

tree = lookup_object(hexsha)

248

230

if base_hexsha is None:

249

231

base_tree = None

250

old_path = None # Newly appeared here

251

232

else:

252

233

base_tree = lookup_object(base_hexsha)

253

old_path = path.decode("utf-8") # Renames aren't supported yet

254

new_path = path.decode("utf-8")

255

if base_tree is None or type(base_tree) is not Tree:

256

ie.revision = revision_id

257

invdelta.append((old_path, new_path, ie.file_id, ie))

258

texts.insert_record_stream([

259

ChunkedContentFactory((ie.file_id, ie.revision), (), None, [])])

234

235

if base_ie is not None and base_ie.kind == "directory":

236

base_children = base_ie.children

237

else:

238

base_children = {}

260

239

# Remember for next time

261

240

existing_children = set()

262

241

child_modes = {}

263

for child_mode, name, child_hexsha in tree.entries():

264

existing_children.add(name)

265

child_path = posixpath.join(path, name)

242

shamap = {}

243

for mode, name, child_hexsha in tree.entries():

244

basename = name.decode("utf-8")

245

existing_children.add(basename)

246

child_path = osutils.pathjoin(path, name)

266

247

if type(base_tree) is Tree:

267

248

try:

268

249

child_base_mode, child_base_hexsha = base_tree[name]

272

253

else:

273

254

child_base_hexsha = None

274

255

child_base_mode = 0

275

if stat.S_ISDIR(child_mode):

276

subinvdelta, grandchildmodes = import_git_tree(texts, mapping,

277

child_path, name, (child_base_hexsha, child_hexsha), base_inv,

278

file_id, revision_id, parent_invs, lookup_object,

279

(child_base_mode, child_mode), store_updater, lookup_file_id,

280

allow_submodules=allow_submodules)

281

elif S_ISGITLINK(child_mode): # submodule

256

if stat.S_ISDIR(mode):

257

subinvdelta, grandchildmodes, subshamap = import_git_tree(

258

texts, mapping, child_path, child_hexsha, child_base_hexsha,

259

base_inv, base_inv_shamap, base_children.get(basename),

260

file_id, revision_id, parent_invs, lookup_object,

261

allow_submodules=allow_submodules)

262

elif S_ISGITLINK(mode): # submodule

282

263

if not allow_submodules:

283

264

raise SubmodulesRequireSubtrees()

284

subinvdelta, grandchildmodes = import_git_submodule(texts, mapping,

285

child_path, name, (child_base_hexsha, child_hexsha), base_inv,

286

file_id, revision_id, parent_invs, lookup_object,

287

(child_base_mode, child_mode), store_updater, lookup_file_id)

265

subinvdelta, grandchildmodes, subshamap = import_git_submodule(

266

texts, mapping, child_path, child_hexsha, child_base_hexsha,

267

base_inv, base_children.get(basename),

268

file_id, revision_id, parent_invs, lookup_object)

288

269

else:

289

subinvdelta = import_git_blob(texts, mapping, child_path, name,

290

(child_base_hexsha, child_hexsha), base_inv, file_id,

291

revision_id, parent_invs, lookup_object,

292

(child_base_mode, child_mode), store_updater, lookup_file_id)

270

subinvdelta, subshamap = import_git_blob(texts, mapping,

271

child_path, child_hexsha, child_base_hexsha,

272

base_inv, base_inv_shamap,

273

base_children.get(basename), file_id,

274

revision_id, parent_invs, lookup_object,

275

mode_is_executable(mode), stat.S_ISLNK(mode))

293

276

grandchildmodes = {}

294

277

child_modes.update(grandchildmodes)

295

278

invdelta.extend(subinvdelta)

296

if child_mode not in (stat.S_IFDIR, DEFAULT_FILE_MODE,

279

shamap.update(subshamap)

280

if mode not in (stat.S_IFDIR, DEFAULT_FILE_MODE,

297

281

stat.S_IFLNK, DEFAULT_FILE_MODE|0111):

298

child_modes[child_path] = child_mode

282

child_modes[child_path] = mode

299

283

# Remove any children that have disappeared

300

if base_tree is not None and type(base_tree) is Tree:

301

invdelta.extend(remove_disappeared_children(base_inv, old_path,

302

base_tree, existing_children, lookup_object))

303

store_updater.add_object(tree, ie, path)

304

return invdelta, child_modes

305

306

307

def verify_commit_reconstruction(target_git_object_retriever, lookup_object,

308

o, rev, ret_tree, parent_trees, mapping, unusual_modes):

309

new_unusual_modes = mapping.export_unusual_file_modes(rev)

310

if new_unusual_modes != unusual_modes:

311

raise AssertionError("unusual modes don't match: %r != %r" % (

312

unusual_modes, new_unusual_modes))

313

# Verify that we can reconstruct the commit properly

314

rec_o = target_git_object_retriever._reconstruct_commit(rev, o.tree, True)

315

if rec_o != o:

316

raise AssertionError("Reconstructed commit differs: %r != %r" % (

317

rec_o, o))

318

diff = []

319

new_objs = {}

320

for path, obj, ie in _tree_to_objects(ret_tree, parent_trees,

321

target_git_object_retriever._cache.idmap, unusual_modes, mapping.BZR_DUMMY_FILE):

322

old_obj_id = tree_lookup_path(lookup_object, o.tree, path)[1]

323

new_objs[path] = obj

324

if obj.id != old_obj_id:

325

diff.append((path, lookup_object(old_obj_id), obj))

326

for (path, old_obj, new_obj) in diff:

327

while (old_obj.type_name == "tree" and

328

new_obj.type_name == "tree" and

329

sorted(old_obj) == sorted(new_obj)):

330

for name in old_obj:

331

if old_obj[name][0] != new_obj[name][0]:

332

raise AssertionError("Modes for %s differ: %o != %o" %

333

(path, old_obj[name][0], new_obj[name][0]))

334

if old_obj[name][1] != new_obj[name][1]:

335

# Found a differing child, delve deeper

336

path = posixpath.join(path, name)

337

old_obj = lookup_object(old_obj[name][1])

338

new_obj = new_objs[path]

339

break

340

raise AssertionError("objects differ for %s: %r != %r" % (path,

341

old_obj, new_obj))

284

if base_ie is not None and base_ie.kind == "directory":

285

invdelta.extend(remove_disappeared_children(base_inv.id2path(file_id),

286

base_children, existing_children))

287

shamap[file_id] = hexsha

288

return invdelta, child_modes, shamap

342

289

343

290

344

291

def import_git_commit(repo, mapping, head, lookup_object,

345

target_git_object_retriever, trees_cache):

292

target_git_object_retriever, parent_invs_cache):

346

293

o = lookup_object(head)

347

rev, roundtrip_revid, verifiers = mapping.import_commit(o,

348

lambda x: target_git_object_retriever.lookup_git_sha(x)[1][0])

294

rev = mapping.import_commit(o)

349

295

# We have to do this here, since we have to walk the tree and

350

296

# we need to make sure to import the blobs / trees with the right

351

297

# path; this may involve adding them more than once.

352

parent_trees = trees_cache.revision_trees(rev.parent_ids)

353

if parent_trees == []:

298

parent_invs = parent_invs_cache.get_inventories(rev.parent_ids)

299

if parent_invs == []:

354

300

base_inv = Inventory(root_id=None)

301

base_ie = None

302

base_inv_shamap = None # Should never be accessed

355

303

base_tree = None

356

base_mode = None

357

304

else:

358

base_inv = parent_trees[0].inventory

305

base_inv = parent_invs[0]

306

base_ie = base_inv.root

307

base_inv_shamap = target_git_object_retriever._idmap.get_inventory_sha_map(base_inv.revision_id)

359

308

base_tree = lookup_object(o.parents[0]).tree

360

base_mode = stat.S_IFDIR

361

store_updater = target_git_object_retriever._get_updater(rev)

362

fileid_map = mapping.get_fileid_map(lookup_object, o.tree)

363

inv_delta, unusual_modes = import_git_tree(repo.texts,

364

mapping, "", "", (base_tree, o.tree), base_inv,

365

None, rev.revision_id, [p.inventory for p in parent_trees],

366

lookup_object, (base_mode, stat.S_IFDIR), store_updater,

367

fileid_map.lookup_file_id,

309

inv_delta, unusual_modes, shamap = import_git_tree(repo.texts,

310

mapping, "", o.tree, base_tree, base_inv, base_inv_shamap,

311

base_ie, None, rev.revision_id, parent_invs, lookup_object,

368

312

allow_submodules=getattr(repo._format, "supports_tree_reference", False))

313

entries = []

314

for (oldpath, newpath, fileid, new_ie) in inv_delta:

315

if newpath is None:

316

entries.append((fileid, None, None, None))

317

else:

318

if new_ie.kind in ("file", "symlink"):

319

entries.append((fileid, "blob", shamap[fileid], new_ie.revision))

320

elif new_ie.kind == "directory":

321

entries.append((fileid, "tree", shamap[fileid], rev.revision_id))

322

else:

323

raise AssertionError

324

target_git_object_retriever._idmap.add_entries(rev.revision_id,

325

rev.parent_ids, head, o.tree, entries)

369

326

if unusual_modes != {}:

370

327

for path, mode in unusual_modes.iteritems():

371

328

warn_unusual_mode(rev.foreign_revid, path, mode)

376

333

basis_id = NULL_REVISION

377

334

base_inv = None

378

335

rev.inventory_sha1, inv = repo.add_inventory_by_delta(basis_id,

379

inv_delta, rev.revision_id, rev.parent_ids, base_inv)

380

# FIXME: Check verifiers

381

testament = StrictTestament3(rev, inv)

382

calculated_verifiers = { "testament3-sha1": testament.as_sha1() }

383

if roundtrip_revid is not None:

384

original_revid = rev.revision_id

385

rev.revision_id = roundtrip_revid

386

if calculated_verifiers != verifiers:

387

trace.mutter("Testament SHA1 %r for %r did not match %r.",

388

calculated_verifiers["testament3-sha1"],

389

rev.revision_id, verifiers["testament3-sha1"])

390

rev.revision_id = original_revid

391

store_updater.add_object(o, calculated_verifiers, None)

392

store_updater.finish()

393

ret_tree = RevisionTree(repo, inv, rev.revision_id)

394

trees_cache.add(ret_tree)

336

inv_delta, rev.revision_id, rev.parent_ids,

337

base_inv)

338

parent_invs_cache.add(rev.revision_id, inv)

395

339

repo.add_revision(rev.revision_id, rev)

396

340

if "verify" in debug.debug_flags:

397

verify_commit_reconstruction(target_git_object_retriever,

398

lookup_object, o, rev, ret_tree, parent_trees, mapping,

399

unusual_modes)

341

new_unusual_modes = mapping.export_unusual_file_modes(rev)

342

if new_unusual_modes != unusual_modes:

343

raise AssertionError("unusual modes don't match: %r != %r" % (unusual_modes, new_unusual_modes))

344

objs = inventory_to_tree_and_blobs(inv, repo.texts, mapping, unusual_modes)

345

for newsha1, newobj, path in objs:

346

assert path is not None

347

if path == "":

348

oldsha1 = o.tree

349

else:

350

(oldmode, oldsha1) = tree_lookup_path(lookup_object, o.tree, path)

351

if oldsha1 != newsha1:

352

raise AssertionError("%r != %r in %s" % (oldsha1, newsha1, path))

400

353

401

354

402

355

def import_git_objects(repo, mapping, object_iter,

413

366

return object_iter[sha]

414

367

except KeyError:

415

368

return target_git_object_retriever[sha]

369

# TODO: a more (memory-)efficient implementation of this

416

370

graph = []

417

371

checked = set()

418

372

heads = list(set(heads))

419

trees_cache = LRUTreeCache(repo)

373

parent_invs_cache = LRUInventoryCache(repo)

374

target_git_object_retriever.start_write_group() # FIXME: try/finally

420

375

# Find and convert commit objects

421

376

while heads:

422

377

if pb is not None:

428

383

except KeyError:

429

384

continue

430

385

if isinstance(o, Commit):

431

rev, roundtrip_revid, verifiers = mapping.import_commit(o,

432

lambda x: None)

433

if (repo.has_revision(rev.revision_id) or

434

(roundtrip_revid and repo.has_revision(roundtrip_revid))):

386

rev = mapping.import_commit(o)

387

if repo.has_revision(rev.revision_id):

435

388

continue

389

squash_revision(repo, rev)

436

390

graph.append((o.id, o.parents))

437

391

heads.extend([p for p in o.parents if p not in checked])

438

392

elif isinstance(o, Tag):

444

398

del checked

445

399

# Order the revisions

446

400

# Create the inventory objects

447

batch_size = 1000

401

batch_size = 100

448

402

revision_ids = topo_sort(graph)

449

403

pack_hints = []

450

404

if limit is not None:

451

405

revision_ids = revision_ids[:limit]

452

406

last_imported = None

453

407

for offset in range(0, len(revision_ids), batch_size):

454

target_git_object_retriever.start_write_group()

408

repo.start_write_group()

455

409

try:

456

repo.start_write_group()

457

try:

458

for i, head in enumerate(

459

revision_ids[offset:offset+batch_size]):

460

if pb is not None:

461

pb.update("fetching revisions", offset+i,

462

len(revision_ids))

463

import_git_commit(repo, mapping, head, lookup_object,

464

target_git_object_retriever, trees_cache)

465

last_imported = head

466

except:

467

repo.abort_write_group()

468

raise

469

else:

470

hint = repo.commit_write_group()

471

if hint is not None:

472

pack_hints.extend(hint)

410

for i, head in enumerate(revision_ids[offset:offset+batch_size]):

411

if pb is not None:

412

pb.update("fetching revisions", offset+i, len(revision_ids))

413

import_git_commit(repo, mapping, head, lookup_object,

414

target_git_object_retriever,

415

parent_invs_cache)

416

last_imported = head

473

417

except:

474

target_git_object_retriever.abort_write_group()

418

repo.abort_write_group()

475

419

raise

476

420

else:

477

target_git_object_retriever.commit_write_group()

421

hint = repo.commit_write_group()

422

if hint is not None:

423

pack_hints.extend(hint)

424

target_git_object_retriever.commit_write_group()

478

425

return pack_hints, last_imported

479

426

480

427

490

437

"""See InterRepository.copy_content."""

491

438

self.fetch(revision_id, pb, find_ghosts=False)

492

439

440

def fetch(self, revision_id=None, pb=None, find_ghosts=False,

441

mapping=None, fetch_spec=None):

442

self.fetch_refs(revision_id=revision_id, pb=pb,

443

find_ghosts=find_ghosts, mapping=mapping, fetch_spec=fetch_spec)

444

493

445

494

446

class InterGitNonGitRepository(InterGitRepository):

495

447

"""Base InterRepository that copies revisions from a Git into a non-Git

496

448

repository."""

497

449

498

def fetch_objects(self, determine_wants, mapping, pb=None, limit=None):

499

"""Fetch objects from a remote server.

500

501

:param determine_wants: determine_wants callback

502

:param mapping: BzrGitMapping to use

503

:param pb: Optional progress bar

504

:param limit: Maximum number of commits to import.

505

:return: Tuple with pack hint, last imported revision id and remote refs

506

"""

507

raise NotImplementedError(self.fetch_objects)

508

509

def fetch(self, revision_id=None, pb=None, find_ghosts=False,

450

def fetch_refs(self, revision_id=None, pb=None, find_ghosts=False,

510

451

mapping=None, fetch_spec=None):

511

452

if mapping is None:

512

453

mapping = self.source.get_mapping()

516

457

interesting_heads = fetch_spec.heads

517

458

else:

518

459

interesting_heads = None

460

self._refs = {}

519

461

def determine_wants(refs):

462

self._refs = refs

520

463

if interesting_heads is None:

521

464

ret = [sha for (ref, sha) in refs.iteritems() if not ref.endswith("^{}")]

522

465

else:

523

ret = [self.source.lookup_bzr_revision_id(revid)[0] for revid in interesting_heads if revid not in (None, NULL_REVISION)]

524

return [rev for rev in ret if not self.target.has_revision(self.source.lookup_foreign_revision_id(rev))]

525

(pack_hint, _, remote_refs) = self.fetch_objects(determine_wants, mapping, pb)

466

ret = [mapping.revision_id_bzr_to_foreign(revid)[0] for revid in interesting_heads if revid not in (None, NULL_REVISION)]

467

return [rev for rev in ret if not self.target.has_revision(mapping.revision_id_foreign_to_bzr(rev))]

468

(pack_hint, _) = self.fetch_objects(determine_wants, mapping, pb)

526

469

if pack_hint is not None and self.target._format.pack_compresses:

527

470

self.target.pack(hint=pack_hint)

528

return remote_refs

471

if interesting_heads is not None:

472

present_interesting_heads = self.target.has_revisions(interesting_heads)

473

missing_interesting_heads = set(interesting_heads) - present_interesting_heads

474

if missing_interesting_heads:

475

raise AssertionError("Missing interesting heads: %r" % missing_interesting_heads)

476

return self._refs

529

477

530

478

531

479

_GIT_PROGRESS_RE = re.compile(r"(.*?): +(\d+)% \((\d+)/(\d+)\)")

539

487

pb.update(text, 0, 0)

540

488

541

489

542

class DetermineWantsRecorder(object):

543

544

def __init__(self, actual):

545

self.actual = actual

546

self.wants = []

547

self.remote_refs = {}

548

549

def __call__(self, refs):

550

self.remote_refs = refs

551

self.wants = self.actual(refs)

552

return self.wants

553

554

555

490

class InterRemoteGitNonGitRepository(InterGitNonGitRepository):

556

491

"""InterRepository that copies revisions from a remote Git into a non-Git

557

492

repository."""

565

500

return set(all_revs) - all_parents

566

501

567

502

def fetch_objects(self, determine_wants, mapping, pb=None, limit=None):

568

"""See `InterGitNonGitRepository`."""

569

503

def progress(text):

570

504

report_git_progress(pb, text)

571

505

store = BazaarObjectStore(self.target, mapping)

574

508

heads = self.get_target_heads()

575

509

graph_walker = store.get_graph_walker(

576

510

[store._lookup_revision_sha1(head) for head in heads])

577

wants_recorder = DetermineWantsRecorder(determine_wants)

511

recorded_wants = []

512

513

def record_determine_wants(heads):

514

wants = determine_wants(heads)

515

recorded_wants.extend(wants)

516

return wants

578

517

579

518

create_pb = None

580

519

if pb is None:

581

520

create_pb = pb = ui.ui_factory.nested_progress_bar()

582

521

try:

583

522

objects_iter = self.source.fetch_objects(

584

wants_recorder, graph_walker, store.get_raw,

585

progress)

586

(pack_hint, last_rev) = import_git_objects(self.target, mapping,

587

objects_iter, store, wants_recorder.wants, pb, limit)

588

return (pack_hint, last_rev, wants_recorder.remote_refs)

523

record_determine_wants, graph_walker,

524

store.get_raw, progress)

525

return import_git_objects(self.target, mapping,

526

objects_iter, store, recorded_wants, pb, limit)

589

527

finally:

590

528

if create_pb:

591

529

create_pb.finished()

595

533

@staticmethod

596

534

def is_compatible(source, target):

597

535

"""Be compatible with GitRepository."""

536

# FIXME: Also check target uses VersionedFile

598

537

return (isinstance(source, RemoteGitRepository) and

599

538

target.supports_rich_root() and

600

not isinstance(target, GitRepository) and

601

target.texts is not None)

539

not isinstance(target, GitRepository))

602

540

603

541

604

542

class InterLocalGitNonGitRepository(InterGitNonGitRepository):

606

544

repository."""

607

545

608

546

def fetch_objects(self, determine_wants, mapping, pb=None, limit=None):

609

"""See `InterGitNonGitRepository`."""

610

remote_refs = self.source._git.get_refs()

611

wants = determine_wants(remote_refs)

547

"""Fetch objects.

548

"""

549

wants = determine_wants(self.source._git.get_refs())

612

550

create_pb = None

613

551

if pb is None:

614

552

create_pb = pb = ui.ui_factory.nested_progress_bar()

616

554

try:

617

555

self.target.lock_write()

618

556

try:

619

(pack_hint, last_rev) = import_git_objects(self.target, mapping,

557

return import_git_objects(self.target, mapping,

620

558

self.source._git.object_store,

621

559

target_git_object_retriever, wants, pb, limit)

622

return (pack_hint, last_rev, remote_refs)

623

560

finally:

624

561

self.target.unlock()

625

562

finally:

629

566

@staticmethod

630

567

def is_compatible(source, target):

631

568

"""Be compatible with GitRepository."""

569

# FIXME: Also check target uses VersionedFile

632

570

return (isinstance(source, LocalGitRepository) and

633

571

target.supports_rich_root() and

634

not isinstance(target, GitRepository) and

635

target.texts is not None)

572

not isinstance(target, GitRepository))

636

573

637

574

638

575

class InterGitGitRepository(InterGitRepository):

644

581

graphwalker = self.target._git.get_graph_walker()

645

582

if (isinstance(self.source, LocalGitRepository) and

646

583

isinstance(self.target, LocalGitRepository)):

647

refs = self.source._git.fetch(self.target._git, determine_wants,

584

return self.source._git.fetch(self.target._git, determine_wants,

648

585

progress)

649

return (None, None, refs)

650

586

elif (isinstance(self.source, LocalGitRepository) and

651

587

isinstance(self.target, RemoteGitRepository)):

652

588

raise NotImplementedError

654

590

isinstance(self.target, LocalGitRepository)):

655

591

f, commit = self.target._git.object_store.add_thin_pack()

656

592

try:

657

refs = self.source.bzrdir.root_transport.fetch_pack(

658

determine_wants, graphwalker, f.write, progress)

593

refs = self.source._git.fetch_pack(determine_wants,

594

graphwalker, f.write, progress)

659

595

commit()

660

return (None, None, refs)

596

return refs

661

597

except:

662

598

f.close()

663

599

raise

664

600

else:

665

601

raise AssertionError

666

602

667

def fetch(self, revision_id=None, pb=None, find_ghosts=False,

603

def fetch_refs(self, revision_id=None, pb=None, find_ghosts=False,

668

604

mapping=None, fetch_spec=None, branches=None):

669

605

if mapping is None:

670

606

mapping = self.source.get_mapping()

679

615

determine_wants = r.object_store.determine_wants_all

680

616

else:

681

617

determine_wants = lambda x: [y for y in args if not y in r.object_store]

682

self.fetch_objects(determine_wants, mapping)

618

return self.fetch_objects(determine_wants, mapping)[0]

619

683

620

684

621

@staticmethod

685

622

def is_compatible(source, target):