/brz/remove-bazaar : revision 0.252.16

1

2

#

3

# This program is free software; you can redistribute it and/or modify

4

# it under the terms of the GNU General Public License as published by

56

from bzrlib.revisiontree import (

57

RevisionTree,

58

)

59

from bzrlib.testament import (

60

StrictTestament3,

61

)

62

59

from bzrlib.tsort import (

63

60

topo_sort,

64

61

)

70

67

DEFAULT_FILE_MODE,

71

68

mode_is_executable,

72

69

mode_kind,

70

squash_revision,

73

71

warn_unusual_mode,

74

72

)

75

73

from bzrlib.plugins.git.object_store import (

98

96

:param blob: A git blob

99

97

:return: Inventory delta for this file

100

98

"""

101

if mapping.is_control_file(path):

102

return []

103

99

if base_hexsha == hexsha and base_mode == mode:

104

100

# If nothing has changed since the base revision, we're done

105

101

return []

109

105

else:

110

106

cls = InventoryFile

111

107

ie = cls(file_id, name.decode("utf-8"), parent_id)

112

if ie.kind == "file":

113

ie.executable = mode_is_executable(mode)

108

ie.executable = mode_is_executable(mode)

114

109

if base_hexsha == hexsha and mode_kind(base_mode) == mode_kind(mode):

115

110

base_ie = base_inv[base_inv.path2id(path)]

116

111

ie.text_size = base_ie.text_size

117

112

ie.text_sha1 = base_ie.text_sha1

118

if ie.kind == "symlink":

119

ie.symlink_target = base_ie.symlink_target

113

ie.symlink_target = base_ie.symlink_target

120

114

if ie.executable == base_ie.executable:

121

115

ie.revision = base_ie.revision

122

116

else:

126

120

if ie.kind == "symlink":

127

121

ie.revision = None

128

122

ie.symlink_target = blob.data

123

ie.text_size = None

124

ie.text_sha1 = None

129

125

else:

130

126

ie.text_size = sum(imap(len, blob.chunked))

131

127

ie.text_sha1 = osutils.sha_strings(blob.chunked)

132

128

# Check what revision we should store

133

129

parent_keys = []

134

for pinv in parent_invs:

130

for pinv in parent_invs[1:]:

135

131

try:

136

132

pie = pinv[file_id]

137

133

except NoSuchId:

138

134

continue

139

if (pie.text_sha1 == ie.text_sha1 and

140

pie.executable == ie.executable and

141

pie.symlink_target == ie.symlink_target):

135

if pie.text_sha1 == ie.text_sha1 and pie.executable == ie.executable and pie.symlink_target == ie.symlink_target:

142

136

# found a revision in one of the parents to use

143

137

ie.revision = pie.revision

144

138

break

145

parent_key = (file_id, pie.revision)

146

if not parent_key in parent_keys:

147

parent_keys.append(parent_key)

139

parent_keys.append((file_id, pie.revision))

148

140

if ie.revision is None:

149

141

# Need to store a new revision

150

142

ie.revision = revision_id

153

145

chunks = []

154

146

else:

155

147

chunks = blob.chunked

156

texts.insert_record_stream([

157

ChunkedContentFactory((file_id, ie.revision),

158

tuple(parent_keys), ie.text_sha1, chunks)])

148

texts.insert_record_stream([ChunkedContentFactory((file_id, ie.revision), tuple(parent_keys), ie.text_sha1, chunks)])

159

149

invdelta = []

160

150

if base_hexsha is not None:

161

old_path = path.decode("utf-8") # Renames are not supported yet

151

old_path = path # Renames are not supported yet

162

152

if stat.S_ISDIR(base_mode):

163

153

invdelta.extend(remove_disappeared_children(base_inv, old_path,

164

154

lookup_object(base_hexsha), [], lookup_object))

165

155

else:

166

156

old_path = None

167

new_path = path.decode("utf-8")

168

invdelta.append((old_path, new_path, file_id, ie))

157

invdelta.append((old_path, path, file_id, ie))

169

158

if base_hexsha != hexsha:

170

store_updater.add_object(blob, ie, path)

159

store_updater.add_object(blob, ie)

171

160

return invdelta

172

161

173

162

189

178

else:

190

179

oldpath = path

191

180

ie.reference_revision = mapping.revision_id_foreign_to_bzr(hexsha)

192

texts.insert_record_stream([

193

ChunkedContentFactory((file_id, ie.revision), (), None, [])])

181

texts.insert_record_stream([ChunkedContentFactory((file_id, ie.revision), (), None, [])])

194

182

invdelta = [(oldpath, path, file_id, ie)]

195

183

return invdelta, {}

196

184

197

185

198

186

def remove_disappeared_children(base_inv, path, base_tree, existing_children,

199

187

lookup_object):

200

"""Generate an inventory delta for removed children.

201

202

:param base_inv: Base inventory against which to generate the

203

inventory delta.

204

:param path: Path to process (unicode)

205

:param base_tree: Git Tree base object

206

:param existing_children: Children that still exist

207

:param lookup_object: Lookup a git object by its SHA1

208

:return: Inventory delta, as list

209

"""

210

assert type(path) is unicode

211

188

ret = []

212

189

for name, mode, hexsha in base_tree.iteritems():

213

190

if name in existing_children:

214

191

continue

215

192

c_path = posixpath.join(path, name.decode("utf-8"))

216

file_id = base_inv.path2id(c_path)

217

assert file_id is not None

218

ret.append((c_path, None, file_id, None))

193

ret.append((c_path, None, base_inv.path2id(c_path), None))

219

194

if stat.S_ISDIR(mode):

220

195

ret.extend(remove_disappeared_children(

221

196

base_inv, c_path, lookup_object(hexsha), [], lookup_object))

229

204

"""Import a git tree object into a bzr repository.

230

205

231

206

:param texts: VersionedFiles object to add to

232

:param path: Path in the tree (str)

233

:param name: Name of the tree (str)

207

:param path: Path in the tree

234

208

:param tree: A git tree object

235

209

:param base_inv: Base inventory against which to return inventory delta

236

210

:return: Inventory delta for this subtree

237

211

"""

238

assert type(path) is str

239

assert type(name) is str

240

212

if base_hexsha == hexsha and base_mode == mode:

241

213

# If nothing has changed since the base revision, we're done

242

214

return [], {}

250

222

old_path = None # Newly appeared here

251

223

else:

252

224

base_tree = lookup_object(base_hexsha)

253

old_path = path.decode("utf-8") # Renames aren't supported yet

254

new_path = path.decode("utf-8")

225

old_path = path # Renames aren't supported yet

255

226

if base_tree is None or type(base_tree) is not Tree:

256

227

ie.revision = revision_id

257

invdelta.append((old_path, new_path, ie.file_id, ie))

258

texts.insert_record_stream([

259

ChunkedContentFactory((ie.file_id, ie.revision), (), None, [])])

228

invdelta.append((old_path, path, ie.file_id, ie))

229

texts.insert_record_stream([ChunkedContentFactory((ie.file_id, ie.revision), (), None, [])])

260

230

# Remember for next time

261

231

existing_children = set()

262

232

child_modes = {}

273

243

child_base_hexsha = None

274

244

child_base_mode = 0

275

245

if stat.S_ISDIR(child_mode):

276

subinvdelta, grandchildmodes = import_git_tree(texts, mapping,

277

child_path, name, (child_base_hexsha, child_hexsha), base_inv,

278

file_id, revision_id, parent_invs, lookup_object,

279

(child_base_mode, child_mode), store_updater, lookup_file_id,

280

allow_submodules=allow_submodules)

246

subinvdelta, grandchildmodes = import_git_tree(

247

texts, mapping, child_path, name,

248

(child_base_hexsha, child_hexsha),

249

base_inv, file_id, revision_id, parent_invs, lookup_object,

250

(child_base_mode, child_mode), store_updater,

251

lookup_file_id, allow_submodules=allow_submodules)

281

252

elif S_ISGITLINK(child_mode): # submodule

282

253

if not allow_submodules:

283

254

raise SubmodulesRequireSubtrees()

284

subinvdelta, grandchildmodes = import_git_submodule(texts, mapping,

285

child_path, name, (child_base_hexsha, child_hexsha), base_inv,

286

file_id, revision_id, parent_invs, lookup_object,

287

(child_base_mode, child_mode), store_updater, lookup_file_id)

255

subinvdelta, grandchildmodes = import_git_submodule(

256

texts, mapping, child_path, name,

257

(child_base_hexsha, child_hexsha),

258

base_inv, file_id, revision_id, parent_invs, lookup_object,

259

(child_base_mode, child_mode), store_updater,

260

lookup_file_id)

288

261

else:

289

subinvdelta = import_git_blob(texts, mapping, child_path, name,

290

(child_base_hexsha, child_hexsha), base_inv, file_id,

291

revision_id, parent_invs, lookup_object,

292

(child_base_mode, child_mode), store_updater, lookup_file_id)

262

subinvdelta = import_git_blob(texts, mapping,

263

child_path, name, (child_base_hexsha, child_hexsha),

264

base_inv, file_id, revision_id, parent_invs, lookup_object,

265

(child_base_mode, child_mode), store_updater,

266

lookup_file_id)

293

267

grandchildmodes = {}

294

268

child_modes.update(grandchildmodes)

295

269

invdelta.extend(subinvdelta)

298

272

child_modes[child_path] = child_mode

299

273

# Remove any children that have disappeared

300

274

if base_tree is not None and type(base_tree) is Tree:

301

invdelta.extend(remove_disappeared_children(base_inv, old_path,

275

invdelta.extend(remove_disappeared_children(base_inv, old_path,

302

276

base_tree, existing_children, lookup_object))

303

store_updater.add_object(tree, ie, path)

277

store_updater.add_object(tree, ie)

304

278

return invdelta, child_modes

305

279

306

280

311

285

raise AssertionError("unusual modes don't match: %r != %r" % (

312

286

unusual_modes, new_unusual_modes))

313

287

# Verify that we can reconstruct the commit properly

314

rec_o = target_git_object_retriever._reconstruct_commit(rev, o.tree, True)

288

rec_o = target_git_object_retriever._reconstruct_commit(rev, o.tree)

315

289

if rec_o != o:

316

290

raise AssertionError("Reconstructed commit differs: %r != %r" % (

317

291

rec_o, o))

318

292

diff = []

319

293

new_objs = {}

320

294

for path, obj, ie in _tree_to_objects(ret_tree, parent_trees,

321

target_git_object_retriever._cache.idmap, unusual_modes, mapping.BZR_DUMMY_FILE):

295

target_git_object_retriever._cache.idmap, unusual_modes):

322

296

old_obj_id = tree_lookup_path(lookup_object, o.tree, path)[1]

323

297

new_objs[path] = obj

324

298

if obj.id != old_obj_id:

329

303

sorted(old_obj) == sorted(new_obj)):

330

304

for name in old_obj:

331

305

if old_obj[name][0] != new_obj[name][0]:

332

raise AssertionError("Modes for %s differ: %o != %o" %

333

(path, old_obj[name][0], new_obj[name][0]))

306

raise AssertionError("Modes for %s differ: %o != %o" % (path, old_obj[name][0], new_obj[name][0]))

334

307

if old_obj[name][1] != new_obj[name][1]:

335

308

# Found a differing child, delve deeper

336

309

path = posixpath.join(path, name)

344

317

def import_git_commit(repo, mapping, head, lookup_object,

345

318

target_git_object_retriever, trees_cache):

346

319

o = lookup_object(head)

347

rev, roundtrip_revid, verifiers = mapping.import_commit(o,

348

lambda x: target_git_object_retriever.lookup_git_sha(x)[1][0])

320

rev, file_ids = mapping.import_commit(o)

349

321

# We have to do this here, since we have to walk the tree and

350

322

# we need to make sure to import the blobs / trees with the right

351

323

# path; this may involve adding them more than once.

359

331

base_tree = lookup_object(o.parents[0]).tree

360

332

base_mode = stat.S_IFDIR

361

333

store_updater = target_git_object_retriever._get_updater(rev)

362

fileid_map = mapping.get_fileid_map(lookup_object, o.tree)

334

store_updater.add_object(o, None)

335

def lookup_file_id(path):

336

try:

337

return file_ids[path]

338

except KeyError:

339

return mapping.generate_file_id(path)

363

340

inv_delta, unusual_modes = import_git_tree(repo.texts,

364

mapping, "", "", (base_tree, o.tree), base_inv,

341

mapping, "", u"", (base_tree, o.tree), base_inv,

365

342

None, rev.revision_id, [p.inventory for p in parent_trees],

366

343

lookup_object, (base_mode, stat.S_IFDIR), store_updater,

367

fileid_map.lookup_file_id,

344

lookup_file_id,

368

345

allow_submodules=getattr(repo._format, "supports_tree_reference", False))

346

store_updater.finish()

369

347

if unusual_modes != {}:

370

348

for path, mode in unusual_modes.iteritems():

371

349

warn_unusual_mode(rev.foreign_revid, path, mode)

376

354

basis_id = NULL_REVISION

377

355

base_inv = None

378

356

rev.inventory_sha1, inv = repo.add_inventory_by_delta(basis_id,

379

inv_delta, rev.revision_id, rev.parent_ids, base_inv)

380

# FIXME: Check verifiers

381

testament = StrictTestament3(rev, inv)

382

calculated_verifiers = { "testament3-sha1": testament.as_sha1() }

383

if roundtrip_revid is not None:

384

original_revid = rev.revision_id

385

rev.revision_id = roundtrip_revid

386

if calculated_verifiers != verifiers:

387

trace.mutter("Testament SHA1 %r for %r did not match %r.",

388

calculated_verifiers["testament3-sha1"],

389

rev.revision_id, verifiers["testament3-sha1"])

390

rev.revision_id = original_revid

391

store_updater.add_object(o, calculated_verifiers, None)

392

store_updater.finish()

357

inv_delta, rev.revision_id, rev.parent_ids,

358

base_inv)

393

359

ret_tree = RevisionTree(repo, inv, rev.revision_id)

394

360

trees_cache.add(ret_tree)

395

361

repo.add_revision(rev.revision_id, rev)

428

394

except KeyError:

429

395

continue

430

396

if isinstance(o, Commit):

431

rev, roundtrip_revid, verifiers = mapping.import_commit(o,

432

lambda x: None)

433

if (repo.has_revision(rev.revision_id) or

434

(roundtrip_revid and repo.has_revision(roundtrip_revid))):

397

rev, file_ids = mapping.import_commit(o)

398

if repo.has_revision(rev.revision_id):

435

399

continue

400

squash_revision(repo, rev)

436

401

graph.append((o.id, o.parents))

437

402

heads.extend([p for p in o.parents if p not in checked])

438

403

elif isinstance(o, Tag):

461

426

pb.update("fetching revisions", offset+i,

462

427

len(revision_ids))

463

428

import_git_commit(repo, mapping, head, lookup_object,

464

target_git_object_retriever, trees_cache)

429

target_git_object_retriever,

430

trees_cache)

465

431

last_imported = head

466

432

except:

467

433

repo.abort_write_group()

490

456

"""See InterRepository.copy_content."""

491

457

self.fetch(revision_id, pb, find_ghosts=False)

492

458

459

def fetch(self, revision_id=None, pb=None, find_ghosts=False,

460

mapping=None, fetch_spec=None):

461

self.fetch_refs(revision_id=revision_id, pb=pb,

462

find_ghosts=find_ghosts, mapping=mapping, fetch_spec=fetch_spec)

463

493

464

494

465

class InterGitNonGitRepository(InterGitRepository):

495

466

"""Base InterRepository that copies revisions from a Git into a non-Git

496

467

repository."""

497

468

498

def fetch_objects(self, determine_wants, mapping, pb=None, limit=None):

499

"""Fetch objects from a remote server.

500

501

:param determine_wants: determine_wants callback

502

:param mapping: BzrGitMapping to use

503

:param pb: Optional progress bar

504

:param limit: Maximum number of commits to import.

505

:return: Tuple with pack hint, last imported revision id and remote refs

506

"""

507

raise NotImplementedError(self.fetch_objects)

508

509

def fetch(self, revision_id=None, pb=None, find_ghosts=False,

469

def fetch_refs(self, revision_id=None, pb=None, find_ghosts=False,

510

470

mapping=None, fetch_spec=None):

511

471

if mapping is None:

512

472

mapping = self.source.get_mapping()

516

476

interesting_heads = fetch_spec.heads

517

477

else:

518

478

interesting_heads = None

479

self._refs = {}

519

480

def determine_wants(refs):

481

self._refs = refs

520

482

if interesting_heads is None:

521

483

ret = [sha for (ref, sha) in refs.iteritems() if not ref.endswith("^{}")]

522

484

else:

523

ret = [self.source.lookup_bzr_revision_id(revid)[0] for revid in interesting_heads if revid not in (None, NULL_REVISION)]

524

return [rev for rev in ret if not self.target.has_revision(self.source.lookup_foreign_revision_id(rev))]

525

(pack_hint, _, remote_refs) = self.fetch_objects(determine_wants, mapping, pb)

485

ret = [mapping.revision_id_bzr_to_foreign(revid)[0] for revid in interesting_heads if revid not in (None, NULL_REVISION)]

486

return [rev for rev in ret if not self.target.has_revision(mapping.revision_id_foreign_to_bzr(rev))]

487

(pack_hint, _) = self.fetch_objects(determine_wants, mapping, pb)

526

488

if pack_hint is not None and self.target._format.pack_compresses:

527

489

self.target.pack(hint=pack_hint)

528

return remote_refs

490

if interesting_heads is not None:

491

present_interesting_heads = self.target.has_revisions(interesting_heads)

492

missing_interesting_heads = set(interesting_heads) - present_interesting_heads

493

if missing_interesting_heads:

494

raise AssertionError("Missing interesting heads: %r" % missing_interesting_heads)

495

return self._refs

529

496

530

497

531

498

_GIT_PROGRESS_RE = re.compile(r"(.*?): +(\d+)% \((\d+)/(\d+)\)")

539

506

pb.update(text, 0, 0)

540

507

541

508

542

class DetermineWantsRecorder(object):

543

544

def __init__(self, actual):

545

self.actual = actual

546

self.wants = []

547

self.remote_refs = {}

548

549

def __call__(self, refs):

550

self.remote_refs = refs

551

self.wants = self.actual(refs)

552

return self.wants

553

554

555

509

class InterRemoteGitNonGitRepository(InterGitNonGitRepository):

556

510

"""InterRepository that copies revisions from a remote Git into a non-Git

557

511

repository."""

565

519

return set(all_revs) - all_parents

566

520

567

521

def fetch_objects(self, determine_wants, mapping, pb=None, limit=None):

568

"""See `InterGitNonGitRepository`."""

569

522

def progress(text):

570

523

report_git_progress(pb, text)

571

524

store = BazaarObjectStore(self.target, mapping)

574

527

heads = self.get_target_heads()

575

528

graph_walker = store.get_graph_walker(

576

529

[store._lookup_revision_sha1(head) for head in heads])

577

wants_recorder = DetermineWantsRecorder(determine_wants)

530

recorded_wants = []

531

532

def record_determine_wants(heads):

533

wants = determine_wants(heads)

534

recorded_wants.extend(wants)

535

return wants

578

536

579

537

create_pb = None

580

538

if pb is None:

581

539

create_pb = pb = ui.ui_factory.nested_progress_bar()

582

540

try:

583

541

objects_iter = self.source.fetch_objects(

584

wants_recorder, graph_walker, store.get_raw,

585

progress)

586

(pack_hint, last_rev) = import_git_objects(self.target, mapping,

587

objects_iter, store, wants_recorder.wants, pb, limit)

588

return (pack_hint, last_rev, wants_recorder.remote_refs)

542

record_determine_wants, graph_walker,

543

store.get_raw, progress)

544

return import_git_objects(self.target, mapping,

545

objects_iter, store, recorded_wants, pb, limit)

589

546

finally:

590

547

if create_pb:

591

548

create_pb.finished()

595

552

@staticmethod

596

553

def is_compatible(source, target):

597

554

"""Be compatible with GitRepository."""

555

# FIXME: Also check target uses VersionedFile

598

556

return (isinstance(source, RemoteGitRepository) and

599

557

target.supports_rich_root() and

600

not isinstance(target, GitRepository) and

601

target.texts is not None)

558

not isinstance(target, GitRepository))

602

559

603

560

604

561

class InterLocalGitNonGitRepository(InterGitNonGitRepository):

606

563

repository."""

607

564

608

565

def fetch_objects(self, determine_wants, mapping, pb=None, limit=None):

609

"""See `InterGitNonGitRepository`."""

610

remote_refs = self.source._git.get_refs()

611

wants = determine_wants(remote_refs)

566

"""Fetch objects.

567

"""

568

wants = determine_wants(self.source._git.get_refs())

612

569

create_pb = None

613

570

if pb is None:

614

571

create_pb = pb = ui.ui_factory.nested_progress_bar()

616

573

try:

617

574

self.target.lock_write()

618

575

try:

619

(pack_hint, last_rev) = import_git_objects(self.target, mapping,

576

return import_git_objects(self.target, mapping,

620

577

self.source._git.object_store,

621

578

target_git_object_retriever, wants, pb, limit)

622

return (pack_hint, last_rev, remote_refs)

623

579

finally:

624

580

self.target.unlock()

625

581

finally:

629

585

@staticmethod

630

586

def is_compatible(source, target):

631

587

"""Be compatible with GitRepository."""

588

# FIXME: Also check target uses VersionedFile

632

589

return (isinstance(source, LocalGitRepository) and

633

590

target.supports_rich_root() and

634

not isinstance(target, GitRepository) and

635

target.texts is not None)

591

not isinstance(target, GitRepository))

636

592

637

593

638

594

class InterGitGitRepository(InterGitRepository):

644

600

graphwalker = self.target._git.get_graph_walker()

645

601

if (isinstance(self.source, LocalGitRepository) and

646

602

isinstance(self.target, LocalGitRepository)):

647

refs = self.source._git.fetch(self.target._git, determine_wants,

603

return self.source._git.fetch(self.target._git, determine_wants,

648

604

progress)

649

return (None, None, refs)

650

605

elif (isinstance(self.source, LocalGitRepository) and

651

606

isinstance(self.target, RemoteGitRepository)):

652

607

raise NotImplementedError

654

609

isinstance(self.target, LocalGitRepository)):

655

610

f, commit = self.target._git.object_store.add_thin_pack()

656

611

try:

657

refs = self.source.bzrdir.root_transport.fetch_pack(

658

determine_wants, graphwalker, f.write, progress)

612

refs = self.source._git.fetch_pack(determine_wants,

613

graphwalker, f.write, progress)

659

614

commit()

660

return (None, None, refs)

615

return refs

661

616

except:

662

617

f.close()

663

618

raise

664

619

else:

665

620

raise AssertionError

666

621

667

def fetch(self, revision_id=None, pb=None, find_ghosts=False,

622

def fetch_refs(self, revision_id=None, pb=None, find_ghosts=False,

668

623

mapping=None, fetch_spec=None, branches=None):

669

624

if mapping is None:

670

625

mapping = self.source.get_mapping()

679

634

determine_wants = r.object_store.determine_wants_all

680

635

else:

681

636

determine_wants = lambda x: [y for y in args if not y in r.object_store]

682

self.fetch_objects(determine_wants, mapping)

637

return self.fetch_objects(determine_wants, mapping)[0]

638

683

639

684

640

@staticmethod

685

641

def is_compatible(source, target):