/brz/remove-bazaar : revision 0.200.557

18

StringIO,

19

)

20

import dulwich as git

21

from dulwich.client import (

22

SimpleFetchGraphWalker,

23

)

24

21

from dulwich.objects import (

25

22

Commit,

26

23

Tag,

27

)

24

S_ISGITLINK,

25

)

26

from dulwich.object_store import (

27

tree_lookup_path,

28

)

29

import stat

28

30

29

31

from bzrlib import (

30

32

debug,

35

37

)

36

38

from bzrlib.errors import (

37

39

InvalidRevisionId,

40

NoSuchId,

38

41

NoSuchRevision,

39

42

)

40

43

from bzrlib.inventory import (

55

58

from bzrlib.tsort import (

56

59

topo_sort,

57

60

)

61

from bzrlib.versionedfile import (

62

FulltextContentFactory,

63

)

58

64

59

from bzrlib.plugins.git.converter import (

60

BazaarObjectStore,

61

)

62

65

from bzrlib.plugins.git.mapping import (

66

DEFAULT_FILE_MODE,

67

inventory_to_tree_and_blobs,

68

mode_is_executable,

69

squash_revision,

63

70

text_to_blob,

71

warn_unusual_mode,

72

)

73

from bzrlib.plugins.git.object_store import (

74

BazaarObjectStore,

75

)

76

from bzrlib.plugins.git.remote import (

77

RemoteGitRepository,

64

78

)

65

79

from bzrlib.plugins.git.repository import (

66

LocalGitRepository,

67

80

GitRepository,

68

81

GitRepositoryFormat,

69

)

70

from bzrlib.plugins.git.remote import (

71

RemoteGitRepository,

72

)

73

74

75

class BzrFetchGraphWalker(object):

76

"""GraphWalker implementation that uses a Bazaar repository."""

77

78

def __init__(self, repository, mapping):

79

self.repository = repository

80

self.mapping = mapping

81

self.done = set()

82

self.heads = set(repository.all_revision_ids())

83

self.parents = {}

84

85

def __iter__(self):

86

return iter(self.next, None)

87

88

def ack(self, sha):

89

revid = self.mapping.revision_id_foreign_to_bzr(sha)

90

self.remove(revid)

91

92

def remove(self, revid):

93

self.done.add(revid)

94

if revid in self.heads:

95

self.heads.remove(revid)

96

if revid in self.parents:

97

for p in self.parents[revid]:

98

self.remove(p)

99

100

def next(self):

101

while self.heads:

102

ret = self.heads.pop()

103

ps = self.repository.get_parent_map([ret])[ret]

104

self.parents[ret] = ps

105

self.heads.update([p for p in ps if not p in self.done])

106

try:

107

self.done.add(ret)

108

return self.mapping.revision_id_bzr_to_foreign(ret)[0]

109

except InvalidRevisionId:

110

pass

111

return None

82

LocalGitRepository,

83

)

112

84

113

85

114

86

def import_git_blob(texts, mapping, path, hexsha, base_inv, parent_id,

126

98

else:

127

99

cls = InventoryFile

128

100

# We just have to hope this is indeed utf-8:

129

ie = cls(file_id, urlutils.basename(path).decode("utf-8"),

130

parent_id)

101

ie = cls(file_id, urlutils.basename(path).decode("utf-8"), parent_id)

131

102

ie.executable = executable

132

103

# See if this has changed at all

133

104

try:

134

base_sha = shagitmap.lookup_blob(file_id, base_inv.revision_id)

135

except KeyError:

105

base_ie = base_inv[file_id]

106

except NoSuchId:

107

base_ie = None

136

108

base_sha = None

137

109

else:

138

if (base_sha == hexsha and base_inv[file_id].executable == ie.executable

139

and base_inv[file_id].kind == ie.kind):

140

# If nothing has changed since the base revision, we're done

141

return []

142

if base_sha == hexsha:

143

ie.text_size = base_inv[file_id].text_size

144

ie.text_sha1 = base_inv[file_id].text_sha1

145

ie.symlink_target = base_inv[file_id].symlink_target

146

ie.revision = base_inv[file_id].revision

110

try:

111

base_sha = shagitmap.lookup_blob(file_id, base_ie.revision)

112

except KeyError:

113

base_sha = None

114

else:

115

if (base_sha == hexsha and base_ie.executable == ie.executable

116

and base_ie.kind == ie.kind):

117

# If nothing has changed since the base revision, we're done

118

return [], []

119

if base_sha == hexsha and base_ie.kind == ie.kind:

120

ie.text_size = base_ie.text_size

121

ie.text_sha1 = base_ie.text_sha1

122

ie.symlink_target = base_ie.symlink_target

123

if ie.executable == base_ie.executable:

124

ie.revision = base_ie.revision

125

else:

126

blob = lookup_object(hexsha)

147

127

else:

148

128

blob = lookup_object(hexsha)

149

129

if ie.kind == "symlink":

130

ie.revision = None

150

131

ie.symlink_target = blob.data

151

132

ie.text_size = None

152

133

ie.text_sha1 = None

156

137

# Check what revision we should store

157

138

parent_keys = []

158

139

for pinv in parent_invs:

159

if not file_id in pinv:

160

continue

161

if pinv[file_id].text_sha1 == ie.text_sha1:

140

if pinv.revision_id == base_inv.revision_id:

141

pie = base_ie

142

if pie is None:

143

continue

144

else:

145

try:

146

pie = pinv[file_id]

147

except NoSuchId:

148

continue

149

if pie.text_sha1 == ie.text_sha1 and pie.executable == ie.executable and pie.symlink_target == ie.symlink_target:

162

150

# found a revision in one of the parents to use

163

ie.revision = pinv[file_id].revision

151

ie.revision = pie.revision

164

152

break

165

parent_keys.append((file_id, pinv[file_id].revision))

153

parent_keys.append((file_id, pie.revision))

166

154

if ie.revision is None:

167

155

# Need to store a new revision

168

156

ie.revision = revision_id

169

157

assert file_id is not None

170

158

assert ie.revision is not None

171

texts.add_lines((file_id, ie.revision), parent_keys,

172

osutils.split_lines(blob.data))

173

if "verify" in debug.debug_flags:

174

assert text_to_blob(blob.data).id == hexsha

175

shagitmap.add_entry(hexsha, "blob", (ie.file_id, ie.revision))

159

texts.insert_record_stream([FulltextContentFactory((file_id, ie.revision), tuple(parent_keys), ie.text_sha1, blob.data)])

160

shamap = [(hexsha, "blob", (ie.file_id, ie.revision))]

161

else:

162

shamap = []

176

163

if file_id in base_inv:

177

164

old_path = base_inv.id2path(file_id)

178

165

else:

179

166

old_path = None

180

return [(old_path, path, file_id, ie)]

167

invdelta = [(old_path, path, file_id, ie)]

168

invdelta.extend(remove_disappeared_children(base_inv, base_ie, []))

169

return (invdelta, shamap)

170

171

172

def import_git_submodule(texts, mapping, path, hexsha, base_inv, parent_id,

173

revision_id, parent_invs, shagitmap, lookup_object):

174

raise NotImplementedError(import_git_submodule)

175

176

177

def remove_disappeared_children(base_inv, base_ie, existing_children):

178

if base_ie is None or base_ie.kind != 'directory':

179

return []

180

ret = []

181

deletable = [v for k,v in base_ie.children.iteritems() if k not in existing_children]

182

while deletable:

183

ie = deletable.pop()

184

ret.append((base_inv.id2path(ie.file_id), None, ie.file_id, None))

185

if ie.kind == "directory":

186

deletable.extend(ie.children.values())

187

return ret

181

188

182

189

183

190

def import_git_tree(texts, mapping, path, hexsha, base_inv, parent_id,

190

197

:param base_inv: Base inventory against which to return inventory delta

191

198

:return: Inventory delta for this subtree

192

199

"""

193

ret = []

200

invdelta = []

194

201

file_id = mapping.generate_file_id(path)

195

202

# We just have to hope this is indeed utf-8:

196

203

ie = InventoryDirectory(file_id, urlutils.basename(path.decode("utf-8")),

197

204

parent_id)

198

if not file_id in base_inv:

205

try:

206

base_ie = base_inv[file_id]

207

except NoSuchId:

199

208

# Newly appeared here

209

base_ie = None

200

210

ie.revision = revision_id

201

texts.add_lines((file_id, ie.revision), [], [])

202

ret.append((None, path, file_id, ie))

211

texts.add_lines((file_id, ie.revision), (), [])

212

invdelta.append((None, path, file_id, ie))

203

213

else:

204

214

# See if this has changed at all

205

215

try:

206

base_sha = shagitmap.lookup_tree(path, base_inv.revision_id)

216

base_sha = shagitmap.lookup_tree(file_id, base_inv.revision_id)

207

217

except KeyError:

208

218

pass

209

219

else:

210

220

if base_sha == hexsha:

211

221

# If nothing has changed since the base revision, we're done

212

return []

222

return [], {}, []

223

if base_ie.kind != "directory":

224

ie.revision = revision_id

225

texts.add_lines((ie.file_id, ie.revision), (), [])

226

invdelta.append((base_inv.id2path(ie.file_id), path, ie.file_id, ie))

213

227

# Remember for next time

214

228

existing_children = set()

215

if "verify" in debug.debug_flags:

216

# FIXME:

217

assert False

218

shagitmap.add_entry(hexsha, "tree", (file_id, revision_id))

229

child_modes = {}

230

shamap = []

219

231

tree = lookup_object(hexsha)

220

for mode, name, hexsha in tree.entries():

221

entry_kind = (mode & 0700000) / 0100000

232

for mode, name, child_hexsha in tree.entries():

222

233

basename = name.decode("utf-8")

223

234

existing_children.add(basename)

224

if path == "":

225

child_path = name

226

else:

227

child_path = urlutils.join(path, name)

228

if entry_kind == 0:

229

ret.extend(import_git_tree(texts, mapping, child_path, hexsha, base_inv,

230

file_id, revision_id, parent_invs, shagitmap, lookup_object))

231

elif entry_kind == 1:

232

fs_mode = mode & 0777

233

file_kind = (mode & 070000) / 010000

234

if file_kind == 0: # regular file

235

symlink = False

236

elif file_kind == 2:

237

symlink = True

238

else:

239

raise AssertionError("Unknown file kind, mode=%r" % (mode,))

240

ret.extend(import_git_blob(texts, mapping, child_path, hexsha, base_inv,

241

file_id, revision_id, parent_invs, shagitmap, lookup_object,

242

bool(fs_mode & 0111), symlink))

243

else:

244

raise AssertionError("Unknown object kind, perms=%r." % (mode,))

235

child_path = osutils.pathjoin(path, name)

236

if stat.S_ISDIR(mode):

237

subinvdelta, grandchildmodes, subshamap = import_git_tree(

238

texts, mapping, child_path, child_hexsha, base_inv,

239

file_id, revision_id, parent_invs, shagitmap, lookup_object)

240

invdelta.extend(subinvdelta)

241

child_modes.update(grandchildmodes)

242

shamap.extend(subshamap)

243

elif S_ISGITLINK(mode): # submodule

244

subinvdelta, grandchildmodes, subshamap = import_git_submodule(

245

texts, mapping, child_path, child_hexsha, base_inv,

246

file_id, revision_id, parent_invs, shagitmap, lookup_object)

247

invdelta.extend(subinvdelta)

248

child_modes.update(grandchildmodes)

249

shamap.extend(subshamap)

250

else:

251

subinvdelta, subshamap = import_git_blob(texts, mapping,

252

child_path, child_hexsha, base_inv, file_id, revision_id,

253

parent_invs, shagitmap, lookup_object,

254

mode_is_executable(mode), stat.S_ISLNK(mode))

255

invdelta.extend(subinvdelta)

256

shamap.extend(subshamap)

257

if mode not in (stat.S_IFDIR, DEFAULT_FILE_MODE,

258

stat.S_IFLNK, DEFAULT_FILE_MODE|0111):

259

child_modes[child_path] = mode

245

260

# Remove any children that have disappeared

246

if file_id in base_inv:

247

deletable = [v for k,v in base_inv[file_id].children.iteritems() if k not in existing_children]

248

while deletable:

249

ie = deletable.pop()

250

ret.append((base_inv.id2path(ie.file_id), None, ie.file_id, None))

251

if ie.kind == "directory":

252

deletable.extend(ie.children.values())

253

return ret

261

invdelta.extend(remove_disappeared_children(base_inv, base_ie, existing_children))

262

shamap.append((hexsha, "tree", (file_id, revision_id)))

263

return invdelta, child_modes, shamap

254

264

255

265

256

266

def import_git_objects(repo, mapping, object_iter, target_git_object_retriever,

257

267

heads, pb=None):

258

268

"""Import a set of git objects into a bzr repository.

259

269

260

:param repo: Bazaar repository

270

:param repo: Target Bazaar repository

261

271

:param mapping: Mapping to use

262

272

:param object_iter: Iterator over Git objects.

263

273

"""

274

def lookup_object(sha):

275

try:

276

return object_iter[sha]

277

except KeyError:

278

return target_git_object_retriever[sha]

264

279

# TODO: a more (memory-)efficient implementation of this

265

280

graph = []

266

281

root_trees = {}

275

290

head = heads.pop()

276

291

assert isinstance(head, str)

277

292

try:

278

o = object_iter[head]

293

o = lookup_object(head)

279

294

except KeyError:

280

295

continue

281

296

if isinstance(o, Commit):

282

297

rev = mapping.import_commit(o)

283

298

if repo.has_revision(rev.revision_id):

284

299

continue

300

squash_revision(repo, rev)

285

301

root_trees[rev.revision_id] = o.tree

286

302

revisions[rev.revision_id] = rev

287

303

graph.append((rev.revision_id, rev.parent_ids))

288

target_git_object_retriever._idmap.add_entry(o.sha().hexdigest(),

289

"commit", (rev.revision_id, o._tree))

304

target_git_object_retriever._idmap.add_entry(o.id, "commit",

305

(rev.revision_id, o.tree))

290

306

heads.extend([p for p in o.parents if p not in checked])

291

307

elif isinstance(o, Tag):

292

308

heads.append(o.object[1])

302

318

# We have to do this here, since we have to walk the tree and

303

319

# we need to make sure to import the blobs / trees with the right

304

320

# path; this may involve adding them more than once.

305

def lookup_object(sha):

306

try:

307

return object_iter[sha]

308

except KeyError:

309

return target_git_object_retriever[sha]

310

321

parent_invs = []

311

322

for parent_id in rev.parent_ids:

312

323

try:

319

330

base_inv = Inventory(root_id=None)

320

331

else:

321

332

base_inv = parent_invs[0]

322

inv_delta = import_git_tree(repo.texts, mapping, "",

323

root_trees[revid], base_inv, None, revid, parent_invs,

324

target_git_object_retriever._idmap, lookup_object)

333

inv_delta, unusual_modes, shamap = import_git_tree(repo.texts,

334

mapping, "", root_trees[revid], base_inv, None, revid,

335

parent_invs, target_git_object_retriever._idmap, lookup_object)

336

target_git_object_retriever._idmap.add_entries(shamap)

337

if unusual_modes != {}:

338

for path, mode in unusual_modes.iteritems():

339

warn_unusual_mode(rev.foreign_revid, path, mode)

340

mapping.import_unusual_file_modes(rev, unusual_modes)

325

341

try:

326

342

basis_id = rev.parent_ids[0]

327

343

except IndexError:

330

346

inv_delta, rev.revision_id, rev.parent_ids)

331

347

parent_invs_cache[rev.revision_id] = inv

332

348

repo.add_revision(rev.revision_id, rev)

349

if "verify" in debug.debug_flags:

350

new_unusual_modes = mapping.export_unusual_file_modes(rev)

351

if new_unusual_modes != unusual_modes:

352

raise AssertionError("unusual modes don't match: %r != %r" % (unusual_modes, new_unusual_modes))

353

objs = inventory_to_tree_and_blobs(inv, repo.texts, mapping, unusual_modes)

354

for sha1, newobj, path in objs:

355

assert path is not None

356

oldobj = tree_lookup_path(lookup_object, root_trees[revid], path)

357

if oldobj != newobj:

358

raise AssertionError("%r != %r in %s" % (oldobj, newobj, path))

359

333

360

target_git_object_retriever._idmap.commit()

334

361

335

362

336

class InterGitNonGitRepository(InterRepository):

337

"""Base InterRepository that copies revisions from a Git into a non-Git

338

repository."""

363

class InterGitRepository(InterRepository):

339

364

340

365

_matching_repo_format = GitRepositoryFormat()

341

366

352

377

self.fetch_refs(revision_id=revision_id, pb=pb, find_ghosts=find_ghosts,

353

378

mapping=mapping, fetch_spec=fetch_spec)

354

379

380

381

class InterGitNonGitRepository(InterGitRepository):

382

"""Base InterRepository that copies revisions from a Git into a non-Git

383

repository."""

384

355

385

def fetch_refs(self, revision_id=None, pb=None, find_ghosts=False,

356

386

mapping=None, fetch_spec=None):

357

387

if mapping is None:

368

398

if interesting_heads is None:

369

399

ret = [sha for (ref, sha) in refs.iteritems() if not ref.endswith("^{}")]

370

400

else:

371

ret = [mapping.revision_id_bzr_to_foreign(revid)[0] for revid in interesting_heads]

401

ret = [mapping.revision_id_bzr_to_foreign(revid)[0] for revid in interesting_heads if revid not in (None, NULL_REVISION)]

372

402

return [rev for rev in ret if not self.target.has_revision(mapping.revision_id_foreign_to_bzr(rev))]

373

403

self.fetch_objects(determine_wants, mapping, pb)

374

404

return self._refs

375

405

376

406

377

378

407

class InterRemoteGitNonGitRepository(InterGitNonGitRepository):

379

408

"""InterRepository that copies revisions from a remote Git into a non-Git

380

409

repository."""

382

411

def fetch_objects(self, determine_wants, mapping, pb=None):

383

412

def progress(text):

384

413

pb.update("git: %s" % text.rstrip("\r\n"), 0, 0)

385

graph_walker = BzrFetchGraphWalker(self.target, mapping)

386

create_pb = None

387

if pb is None:

388

create_pb = pb = ui.ui_factory.nested_progress_bar()

389

target_git_object_retriever = BazaarObjectStore(self.target, mapping)

390

recorded_wants = []

414

store = BazaarObjectStore(self.target, mapping)

415

self.target.lock_write()

416

try:

417

heads = self.target.get_graph().heads(self.target.all_revision_ids())

418

graph_walker = store.get_graph_walker(

419

[store._lookup_revision_sha1(head) for head in heads])

420

recorded_wants = []

391

421

392

def record_determine_wants(heads):

393

wants = determine_wants(heads)

394

recorded_wants.extend(wants)

395

return wants

422

def record_determine_wants(heads):

423

wants = determine_wants(heads)

424

recorded_wants.extend(wants)

425

return wants

396

426

397

try:

398

self.target.lock_write()

427

create_pb = None

428

if pb is None:

429

create_pb = pb = ui.ui_factory.nested_progress_bar()

399

430

try:

400

431

self.target.start_write_group()

401

432

try:

402

433

objects_iter = self.source.fetch_objects(

403

record_determine_wants,

404

graph_walker,

405

target_git_object_retriever.get_raw,

406

progress)

434

record_determine_wants, graph_walker,

435

store.get_raw, progress)

407

436

import_git_objects(self.target, mapping, objects_iter,

408

target_git_object_retriever, recorded_wants, pb)

437

store, recorded_wants, pb)

409

438

finally:

410

439

self.target.commit_write_group()

411

440

finally:

412

self.target.unlock()

441

if create_pb:

442

create_pb.finished()

413

443

finally:

414

if create_pb:

415

create_pb.finished()

444

self.target.unlock()

416

445

417

446

@staticmethod

418

447

def is_compatible(source, target):

424

453

425

454

426

455

class InterLocalGitNonGitRepository(InterGitNonGitRepository):

427

"""InterRepository that copies revisions from a remote Git into a non-Git

456

"""InterRepository that copies revisions from a local Git into a non-Git

428

457

repository."""

429

458

430

459

def fetch_objects(self, determine_wants, mapping, pb=None):

458

487

not isinstance(target, GitRepository))

459

488

460

489

461

class InterGitRepository(InterRepository):

490

class InterGitGitRepository(InterGitRepository):

462

491

"""InterRepository that copies between Git repositories."""

463

492

464

_matching_repo_format = GitRepositoryFormat()

465

466

@staticmethod

467

def _get_repo_format_to_test():

468

return None

469

470

def copy_content(self, revision_id=None, pb=None):

471

"""See InterRepository.copy_content."""

472

self.fetch(revision_id, pb, find_ghosts=False)

473

474

def fetch(self, revision_id=None, pb=None, find_ghosts=False,

475

mapping=None, fetch_spec=None):

493

def fetch_refs(self, revision_id=None, pb=None, find_ghosts=False,

494

mapping=None, fetch_spec=None, branches=None):

476

495

if mapping is None:

477

496

mapping = self.source.get_mapping()

478

497

def progress(text):

482

501

args = [mapping.revision_id_bzr_to_foreign(revision_id)[0]]

483

502

elif fetch_spec is not None:

484

503

args = [mapping.revision_id_bzr_to_foreign(revid)[0] for revid in fetch_spec.heads]

485

if fetch_spec is None and revision_id is None:

504

if branches is not None:

505

determine_wants = lambda x: [x[y] for y in branches if not x[y] in r.object_store]

506

elif fetch_spec is None and revision_id is None:

486

507

determine_wants = r.object_store.determine_wants_all

487

508

else:

488

509

determine_wants = lambda x: [y for y in args if not y in r.object_store]

489

510

490

graphwalker = SimpleFetchGraphWalker(r.heads().values(), r.get_parents)

491

f, commit = r.object_store.add_pack()

511

graphwalker = r.get_graph_walker()

512

f, commit = r.object_store.add_thin_pack()

492

513

try:

493

self.source._git.fetch_pack(path, determine_wants, graphwalker, f.write, progress)

494

f.close()

514

refs = self.source.fetch_pack(determine_wants, graphwalker,

515

f.write, progress)

495

516

commit()

517

return refs

496

518

except:

497

519

f.close()

498

520

raise