/brz/remove-bazaar : revision 0.200.557

18

StringIO,

19

)

20

import dulwich as git

21

from dulwich.client import (

22

SimpleFetchGraphWalker,

23

)

24

21

from dulwich.objects import (

25

22

Commit,

26

23

Tag,

27

)

24

S_ISGITLINK,

25

)

26

from dulwich.object_store import (

27

tree_lookup_path,

28

)

29

import stat

28

30

29

31

from bzrlib import (

32

debug,

30

33

osutils,

31

34

trace,

32

35

ui,

34

37

)

35

38

from bzrlib.errors import (

36

39

InvalidRevisionId,

40

NoSuchId,

37

41

NoSuchRevision,

38

42

)

39

43

from bzrlib.inventory import (

54

58

from bzrlib.tsort import (

55

59

topo_sort,

56

60

)

61

from bzrlib.versionedfile import (

62

FulltextContentFactory,

63

)

57

64

58

from bzrlib.plugins.git.converter import (

65

from bzrlib.plugins.git.mapping import (

66

DEFAULT_FILE_MODE,

67

inventory_to_tree_and_blobs,

68

mode_is_executable,

69

squash_revision,

70

text_to_blob,

71

warn_unusual_mode,

72

)

73

from bzrlib.plugins.git.object_store import (

59

74

BazaarObjectStore,

60

75

)

76

from bzrlib.plugins.git.remote import (

77

RemoteGitRepository,

78

)

61

79

from bzrlib.plugins.git.repository import (

62

LocalGitRepository,

63

80

GitRepository,

64

81

GitRepositoryFormat,

65

)

66

from bzrlib.plugins.git.remote import (

67

RemoteGitRepository,

68

)

69

70

71

class BzrFetchGraphWalker(object):

72

"""GraphWalker implementation that uses a Bazaar repository."""

73

74

def __init__(self, repository, mapping):

75

self.repository = repository

76

self.mapping = mapping

77

self.done = set()

78

self.heads = set(repository.all_revision_ids())

79

self.parents = {}

80

81

def __iter__(self):

82

return iter(self.next, None)

83

84

def ack(self, sha):

85

revid = self.mapping.revision_id_foreign_to_bzr(sha)

86

self.remove(revid)

87

88

def remove(self, revid):

89

self.done.add(revid)

90

if revid in self.heads:

91

self.heads.remove(revid)

92

if revid in self.parents:

93

for p in self.parents[revid]:

94

self.remove(p)

95

96

def next(self):

97

while self.heads:

98

ret = self.heads.pop()

99

ps = self.repository.get_parent_map([ret])[ret]

100

self.parents[ret] = ps

101

self.heads.update([p for p in ps if not p in self.done])

102

try:

103

self.done.add(ret)

104

return self.mapping.revision_id_bzr_to_foreign(ret)[0]

105

except InvalidRevisionId:

106

pass

107

return None

82

LocalGitRepository,

83

)

108

84

109

85

110

86

def import_git_blob(texts, mapping, path, hexsha, base_inv, parent_id,

122

98

else:

123

99

cls = InventoryFile

124

100

# We just have to hope this is indeed utf-8:

125

ie = cls(file_id, urlutils.basename(path).decode("utf-8"),

126

parent_id)

101

ie = cls(file_id, urlutils.basename(path).decode("utf-8"), parent_id)

127

102

ie.executable = executable

128

103

# See if this has changed at all

129

104

try:

130

base_sha = shagitmap.lookup_blob(file_id, base_inv.revision_id)

131

except KeyError:

105

base_ie = base_inv[file_id]

106

except NoSuchId:

107

base_ie = None

132

108

base_sha = None

133

109

else:

134

if (base_sha == hexsha and base_inv[file_id].executable == ie.executable

135

and base_inv[file_id].kind == ie.kind):

136

# If nothing has changed since the base revision, we're done

137

return []

138

if base_sha == hexsha:

139

ie.text_size = base_inv[file_id].text_size

140

ie.text_sha1 = base_inv[file_id].text_sha1

141

ie.symlink_target = base_inv[file_id].symlink_target

142

ie.revision = base_inv[file_id].revision

110

try:

111

base_sha = shagitmap.lookup_blob(file_id, base_ie.revision)

112

except KeyError:

113

base_sha = None

114

else:

115

if (base_sha == hexsha and base_ie.executable == ie.executable

116

and base_ie.kind == ie.kind):

117

# If nothing has changed since the base revision, we're done

118

return [], []

119

if base_sha == hexsha and base_ie.kind == ie.kind:

120

ie.text_size = base_ie.text_size

121

ie.text_sha1 = base_ie.text_sha1

122

ie.symlink_target = base_ie.symlink_target

123

if ie.executable == base_ie.executable:

124

ie.revision = base_ie.revision

125

else:

126

blob = lookup_object(hexsha)

143

127

else:

144

128

blob = lookup_object(hexsha)

145

129

if ie.kind == "symlink":

130

ie.revision = None

146

131

ie.symlink_target = blob.data

147

132

ie.text_size = None

148

133

ie.text_sha1 = None

152

137

# Check what revision we should store

153

138

parent_keys = []

154

139

for pinv in parent_invs:

155

if not file_id in pinv:

156

continue

157

if pinv[file_id].text_sha1 == ie.text_sha1:

140

if pinv.revision_id == base_inv.revision_id:

141

pie = base_ie

142

if pie is None:

143

continue

144

else:

145

try:

146

pie = pinv[file_id]

147

except NoSuchId:

148

continue

149

if pie.text_sha1 == ie.text_sha1 and pie.executable == ie.executable and pie.symlink_target == ie.symlink_target:

158

150

# found a revision in one of the parents to use

159

ie.revision = pinv[file_id].revision

151

ie.revision = pie.revision

160

152

break

161

parent_keys.append((file_id, pinv[file_id].revision))

153

parent_keys.append((file_id, pie.revision))

162

154

if ie.revision is None:

163

155

# Need to store a new revision

164

156

ie.revision = revision_id

165

157

assert file_id is not None

166

158

assert ie.revision is not None

167

texts.add_lines((file_id, ie.revision), parent_keys,

168

osutils.split_lines(blob.data))

169

shagitmap.add_entry(hexsha, "blob", (ie.file_id, ie.revision))

159

texts.insert_record_stream([FulltextContentFactory((file_id, ie.revision), tuple(parent_keys), ie.text_sha1, blob.data)])

160

shamap = [(hexsha, "blob", (ie.file_id, ie.revision))]

161

else:

162

shamap = []

170

163

if file_id in base_inv:

171

164

old_path = base_inv.id2path(file_id)

172

165

else:

173

166

old_path = None

174

return [(old_path, path, file_id, ie)]

167

invdelta = [(old_path, path, file_id, ie)]

168

invdelta.extend(remove_disappeared_children(base_inv, base_ie, []))

169

return (invdelta, shamap)

170

171

172

def import_git_submodule(texts, mapping, path, hexsha, base_inv, parent_id,

173

revision_id, parent_invs, shagitmap, lookup_object):

174

raise NotImplementedError(import_git_submodule)

175

176

177

def remove_disappeared_children(base_inv, base_ie, existing_children):

178

if base_ie is None or base_ie.kind != 'directory':

179

return []

180

ret = []

181

deletable = [v for k,v in base_ie.children.iteritems() if k not in existing_children]

182

while deletable:

183

ie = deletable.pop()

184

ret.append((base_inv.id2path(ie.file_id), None, ie.file_id, None))

185

if ie.kind == "directory":

186

deletable.extend(ie.children.values())

187

return ret

175

188

176

189

177

190

def import_git_tree(texts, mapping, path, hexsha, base_inv, parent_id,

184

197

:param base_inv: Base inventory against which to return inventory delta

185

198

:return: Inventory delta for this subtree

186

199

"""

187

ret = []

200

invdelta = []

188

201

file_id = mapping.generate_file_id(path)

189

202

# We just have to hope this is indeed utf-8:

190

203

ie = InventoryDirectory(file_id, urlutils.basename(path.decode("utf-8")),

191

204

parent_id)

192

if not file_id in base_inv:

205

try:

206

base_ie = base_inv[file_id]

207

except NoSuchId:

193

208

# Newly appeared here

209

base_ie = None

194

210

ie.revision = revision_id

195

texts.add_lines((file_id, ie.revision), [], [])

196

ret.append((None, path, file_id, ie))

211

texts.add_lines((file_id, ie.revision), (), [])

212

invdelta.append((None, path, file_id, ie))

197

213

else:

198

214

# See if this has changed at all

199

215

try:

200

base_sha = shagitmap.lookup_tree(path, base_inv.revision_id)

216

base_sha = shagitmap.lookup_tree(file_id, base_inv.revision_id)

201

217

except KeyError:

202

218

pass

203

219

else:

204

220

if base_sha == hexsha:

205

221

# If nothing has changed since the base revision, we're done

206

return []

222

return [], {}, []

223

if base_ie.kind != "directory":

224

ie.revision = revision_id

225

texts.add_lines((ie.file_id, ie.revision), (), [])

226

invdelta.append((base_inv.id2path(ie.file_id), path, ie.file_id, ie))

207

227

# Remember for next time

208

228

existing_children = set()

209

shagitmap.add_entry(hexsha, "tree", (file_id, revision_id))

229

child_modes = {}

230

shamap = []

210

231

tree = lookup_object(hexsha)

211

for mode, name, hexsha in tree.entries():

212

entry_kind = (mode & 0700000) / 0100000

232

for mode, name, child_hexsha in tree.entries():

213

233

basename = name.decode("utf-8")

214

234

existing_children.add(basename)

215

if path == "":

216

child_path = name

217

else:

218

child_path = urlutils.join(path, name)

219

if entry_kind == 0:

220

ret.extend(import_git_tree(texts, mapping, child_path, hexsha, base_inv,

221

file_id, revision_id, parent_invs, shagitmap, lookup_object))

222

elif entry_kind == 1:

223

fs_mode = mode & 0777

224

file_kind = (mode & 070000) / 010000

225

if file_kind == 0: # regular file

226

symlink = False

227

elif file_kind == 2:

228

symlink = True

229

else:

230

raise AssertionError("Unknown file kind, mode=%r" % (mode,))

231

ret.extend(import_git_blob(texts, mapping, child_path, hexsha, base_inv,

232

file_id, revision_id, parent_invs, shagitmap, lookup_object,

233

bool(fs_mode & 0111), symlink))

234

else:

235

raise AssertionError("Unknown object kind, perms=%r." % (mode,))

235

child_path = osutils.pathjoin(path, name)

236

if stat.S_ISDIR(mode):

237

subinvdelta, grandchildmodes, subshamap = import_git_tree(

238

texts, mapping, child_path, child_hexsha, base_inv,

239

file_id, revision_id, parent_invs, shagitmap, lookup_object)

240

invdelta.extend(subinvdelta)

241

child_modes.update(grandchildmodes)

242

shamap.extend(subshamap)

243

elif S_ISGITLINK(mode): # submodule

244

subinvdelta, grandchildmodes, subshamap = import_git_submodule(

245

texts, mapping, child_path, child_hexsha, base_inv,

246

file_id, revision_id, parent_invs, shagitmap, lookup_object)

247

invdelta.extend(subinvdelta)

248

child_modes.update(grandchildmodes)

249

shamap.extend(subshamap)

250

else:

251

subinvdelta, subshamap = import_git_blob(texts, mapping,

252

child_path, child_hexsha, base_inv, file_id, revision_id,

253

parent_invs, shagitmap, lookup_object,

254

mode_is_executable(mode), stat.S_ISLNK(mode))

255

invdelta.extend(subinvdelta)

256

shamap.extend(subshamap)

257

if mode not in (stat.S_IFDIR, DEFAULT_FILE_MODE,

258

stat.S_IFLNK, DEFAULT_FILE_MODE|0111):

259

child_modes[child_path] = mode

236

260

# Remove any children that have disappeared

237

if file_id in base_inv:

238

deletable = [v for k,v in base_inv[file_id].children.iteritems() if k not in existing_children]

239

while deletable:

240

ie = deletable.pop()

241

ret.append((base_inv.id2path(ie.file_id), None, ie.file_id, None))

242

if ie.kind == "directory":

243

deletable.extend(ie.children.values())

244

return ret

261

invdelta.extend(remove_disappeared_children(base_inv, base_ie, existing_children))

262

shamap.append((hexsha, "tree", (file_id, revision_id)))

263

return invdelta, child_modes, shamap

245

264

246

265

247

266

def import_git_objects(repo, mapping, object_iter, target_git_object_retriever,

248

267

heads, pb=None):

249

268

"""Import a set of git objects into a bzr repository.

250

269

251

:param repo: Bazaar repository

270

:param repo: Target Bazaar repository

252

271

:param mapping: Mapping to use

253

272

:param object_iter: Iterator over Git objects.

254

273

"""

274

def lookup_object(sha):

275

try:

276

return object_iter[sha]

277

except KeyError:

278

return target_git_object_retriever[sha]

255

279

# TODO: a more (memory-)efficient implementation of this

256

280

graph = []

257

281

root_trees = {}

266

290

head = heads.pop()

267

291

assert isinstance(head, str)

268

292

try:

269

o = object_iter[head]

293

o = lookup_object(head)

270

294

except KeyError:

271

295

continue

272

296

if isinstance(o, Commit):

273

297

rev = mapping.import_commit(o)

274

298

if repo.has_revision(rev.revision_id):

275

299

continue

300

squash_revision(repo, rev)

276

301

root_trees[rev.revision_id] = o.tree

277

302

revisions[rev.revision_id] = rev

278

303

graph.append((rev.revision_id, rev.parent_ids))

279

target_git_object_retriever._idmap.add_entry(o.sha().hexdigest(),

280

"commit", (rev.revision_id, o._tree))

304

target_git_object_retriever._idmap.add_entry(o.id, "commit",

305

(rev.revision_id, o.tree))

281

306

heads.extend([p for p in o.parents if p not in checked])

282

307

elif isinstance(o, Tag):

283

308

heads.append(o.object[1])

293

318

# We have to do this here, since we have to walk the tree and

294

319

# we need to make sure to import the blobs / trees with the right

295

320

# path; this may involve adding them more than once.

296

def lookup_object(sha):

297

try:

298

return object_iter[sha]

299

except KeyError:

300

return target_git_object_retriever[sha]

301

321

parent_invs = []

302

322

for parent_id in rev.parent_ids:

303

323

try:

310

330

base_inv = Inventory(root_id=None)

311

331

else:

312

332

base_inv = parent_invs[0]

313

inv_delta = import_git_tree(repo.texts, mapping, "",

314

root_trees[revid], base_inv, None, revid, parent_invs,

315

target_git_object_retriever._idmap, lookup_object)

333

inv_delta, unusual_modes, shamap = import_git_tree(repo.texts,

334

mapping, "", root_trees[revid], base_inv, None, revid,

335

parent_invs, target_git_object_retriever._idmap, lookup_object)

336

target_git_object_retriever._idmap.add_entries(shamap)

337

if unusual_modes != {}:

338

for path, mode in unusual_modes.iteritems():

339

warn_unusual_mode(rev.foreign_revid, path, mode)

340

mapping.import_unusual_file_modes(rev, unusual_modes)

316

341

try:

317

342

basis_id = rev.parent_ids[0]

318

343

except IndexError:

321

346

inv_delta, rev.revision_id, rev.parent_ids)

322

347

parent_invs_cache[rev.revision_id] = inv

323

348

repo.add_revision(rev.revision_id, rev)

349

if "verify" in debug.debug_flags:

350

new_unusual_modes = mapping.export_unusual_file_modes(rev)

351

if new_unusual_modes != unusual_modes:

352

raise AssertionError("unusual modes don't match: %r != %r" % (unusual_modes, new_unusual_modes))

353

objs = inventory_to_tree_and_blobs(inv, repo.texts, mapping, unusual_modes)

354

for sha1, newobj, path in objs:

355

assert path is not None

356

oldobj = tree_lookup_path(lookup_object, root_trees[revid], path)

357

if oldobj != newobj:

358

raise AssertionError("%r != %r in %s" % (oldobj, newobj, path))

359

324

360

target_git_object_retriever._idmap.commit()

325

361

326

362

327

class InterGitNonGitRepository(InterRepository):

328

"""Base InterRepository that copies revisions from a Git into a non-Git

329

repository."""

363

class InterGitRepository(InterRepository):

330

364

331

365

_matching_repo_format = GitRepositoryFormat()

332

366

343

377

self.fetch_refs(revision_id=revision_id, pb=pb, find_ghosts=find_ghosts,

344

378

mapping=mapping, fetch_spec=fetch_spec)

345

379

380

381

class InterGitNonGitRepository(InterGitRepository):

382

"""Base InterRepository that copies revisions from a Git into a non-Git

383

repository."""

384

346

385

def fetch_refs(self, revision_id=None, pb=None, find_ghosts=False,

347

386

mapping=None, fetch_spec=None):

348

387

if mapping is None:

359

398

if interesting_heads is None:

360

399

ret = [sha for (ref, sha) in refs.iteritems() if not ref.endswith("^{}")]

361

400

else:

362

ret = [mapping.revision_id_bzr_to_foreign(revid)[0] for revid in interesting_heads]

401

ret = [mapping.revision_id_bzr_to_foreign(revid)[0] for revid in interesting_heads if revid not in (None, NULL_REVISION)]

363

402

return [rev for rev in ret if not self.target.has_revision(mapping.revision_id_foreign_to_bzr(rev))]

364

403

self.fetch_objects(determine_wants, mapping, pb)

365

404

return self._refs

366

405

367

406

368

369

407

class InterRemoteGitNonGitRepository(InterGitNonGitRepository):

370

408

"""InterRepository that copies revisions from a remote Git into a non-Git

371

409

repository."""

373

411

def fetch_objects(self, determine_wants, mapping, pb=None):

374

412

def progress(text):

375

413

pb.update("git: %s" % text.rstrip("\r\n"), 0, 0)

376

graph_walker = BzrFetchGraphWalker(self.target, mapping)

377

create_pb = None

378

if pb is None:

379

create_pb = pb = ui.ui_factory.nested_progress_bar()

380

target_git_object_retriever = BazaarObjectStore(self.target, mapping)

381

recorded_wants = []

414

store = BazaarObjectStore(self.target, mapping)

415

self.target.lock_write()

416

try:

417

heads = self.target.get_graph().heads(self.target.all_revision_ids())

418

graph_walker = store.get_graph_walker(

419

[store._lookup_revision_sha1(head) for head in heads])

420

recorded_wants = []

382

421

383

def record_determine_wants(heads):

384

wants = determine_wants(heads)

385

recorded_wants.extend(wants)

386

return wants

422

def record_determine_wants(heads):

423

wants = determine_wants(heads)

424

recorded_wants.extend(wants)

425

return wants

387

426

388

try:

389

self.target.lock_write()

427

create_pb = None

428

if pb is None:

429

create_pb = pb = ui.ui_factory.nested_progress_bar()

390

430

try:

391

431

self.target.start_write_group()

392

432

try:

393

433

objects_iter = self.source.fetch_objects(

394

record_determine_wants,

395

graph_walker,

396

target_git_object_retriever.get_raw,

397

progress)

434

record_determine_wants, graph_walker,

435

store.get_raw, progress)

398

436

import_git_objects(self.target, mapping, objects_iter,

399

target_git_object_retriever, recorded_wants, pb)

437

store, recorded_wants, pb)

400

438

finally:

401

439

self.target.commit_write_group()

402

440

finally:

403

self.target.unlock()

441

if create_pb:

442

create_pb.finished()

404

443

finally:

405

if create_pb:

406

create_pb.finished()

444

self.target.unlock()

407

445

408

446

@staticmethod

409

447

def is_compatible(source, target):

415

453

416

454

417

455

class InterLocalGitNonGitRepository(InterGitNonGitRepository):

418

"""InterRepository that copies revisions from a remote Git into a non-Git

456

"""InterRepository that copies revisions from a local Git into a non-Git

419

457

repository."""

420

458

421

459

def fetch_objects(self, determine_wants, mapping, pb=None):

449

487

not isinstance(target, GitRepository))

450

488

451

489

452

class InterGitRepository(InterRepository):

490

class InterGitGitRepository(InterGitRepository):

453

491

"""InterRepository that copies between Git repositories."""

454

492

455

_matching_repo_format = GitRepositoryFormat()

456

457

@staticmethod

458

def _get_repo_format_to_test():

459

return None

460

461

def copy_content(self, revision_id=None, pb=None):

462

"""See InterRepository.copy_content."""

463

self.fetch(revision_id, pb, find_ghosts=False)

464

465

def fetch(self, revision_id=None, pb=None, find_ghosts=False,

466

mapping=None, fetch_spec=None):

493

def fetch_refs(self, revision_id=None, pb=None, find_ghosts=False,

494

mapping=None, fetch_spec=None, branches=None):

467

495

if mapping is None:

468

496

mapping = self.source.get_mapping()

469

497

def progress(text):

473

501

args = [mapping.revision_id_bzr_to_foreign(revision_id)[0]]

474

502

elif fetch_spec is not None:

475

503

args = [mapping.revision_id_bzr_to_foreign(revid)[0] for revid in fetch_spec.heads]

476

if fetch_spec is None and revision_id is None:

504

if branches is not None:

505

determine_wants = lambda x: [x[y] for y in branches if not x[y] in r.object_store]

506

elif fetch_spec is None and revision_id is None:

477

507

determine_wants = r.object_store.determine_wants_all

478

508

else:

479

509

determine_wants = lambda x: [y for y in args if not y in r.object_store]

480

510

481

graphwalker = SimpleFetchGraphWalker(r.heads().values(), r.get_parents)

482

f, commit = r.object_store.add_pack()

511

graphwalker = r.get_graph_walker()

512

f, commit = r.object_store.add_thin_pack()

483

513

try:

484

self.source._git.fetch_pack(path, determine_wants, graphwalker, f.write, progress)

485

f.close()

514

refs = self.source.fetch_pack(determine_wants, graphwalker,

515

f.write, progress)

486

516

commit()

517

return refs

487

518

except:

488

519

f.close()

489

520

raise