/brz/remove-bazaar : revision 0.239.10

27

28

from bzrlib import (

29

debug,

30

lru_cache,

31

30

osutils,

32

31

trace,

33

32

ui,

42

41

InventoryDirectory,

43

42

InventoryFile,

44

43

InventoryLink,

45

TreeReference,

44

)

45

from bzrlib.lru_cache import (

46

LRUCache,

46

47

)

47

48

from bzrlib.repository import (

48

49

InterRepository,

71

72

RemoteGitRepository,

72

73

)

73

74

from bzrlib.plugins.git.repository import (

74

GitRepository,

75

GitRepository,

75

76

GitRepositoryFormat,

76

77

LocalGitRepository,

77

78

)

78

79

80

MAX_INV_CACHE_SIZE = 50 * 1024 * 1024

81

82

83

def import_git_blob(texts, mapping, path, hexsha, base_inv, base_ie, parent_id,

81

def import_git_blob(texts, mapping, path, hexsha, base_inv, base_ie, parent_id,

84

82

revision_id, parent_invs, shagitmap, lookup_object, executable, symlink):

85

83

"""Import a git blob object into a bzr repository.

86

84

150

148

ie.revision = revision_id

151

149

assert file_id is not None

152

150

assert ie.revision is not None

153

if ie.kind == 'symlink':

154

data = ''

155

else:

156

data = blob.data

157

texts.insert_record_stream([FulltextContentFactory((file_id, ie.revision), tuple(parent_keys), ie.text_sha1, data)])

151

texts.insert_record_stream([FulltextContentFactory((file_id, ie.revision), tuple(parent_keys), ie.text_sha1, blob.data)])

158

152

shamap = [(hexsha, "blob", (ie.file_id, ie.revision))]

159

153

else:

160

154

shamap = []

161

155

invdelta = []

162

if base_ie is not None:

156

if base_ie is not None:

163

157

old_path = base_inv.id2path(file_id)

164

158

if base_ie.kind == "directory":

165

159

invdelta.extend(remove_disappeared_children(old_path, base_ie.children, []))

169

163

return (invdelta, shamap)

170

164

171

165

172

class SubmodulesRequireSubtrees(BzrError):

173

_fmt = """The repository you are fetching from contains submodules. To continue, upgrade your Bazaar repository to a format that supports nested trees, such as 'development-subtree'."""

166

class SubmodulesNotSupported(BzrError):

167

168

_fmt = """Submodules can not yet be imported (requires nested tree support in Bazaar)."""

174

169

internal = False

175

170

176

171

177

def import_git_submodule(texts, mapping, path, hexsha, base_inv, base_ie,

172

def import_git_submodule(texts, mapping, path, hexsha, base_inv, base_ie,

178

173

parent_id, revision_id, parent_invs, shagitmap, lookup_object):

179

file_id = mapping.generate_file_id(path)

180

ie = TreeReference(file_id, urlutils.basename(path.decode("utf-8")),

181

parent_id)

182

ie.revision = revision_id

183

if base_ie is None:

184

oldpath = None

185

else:

186

oldpath = path

187

if (base_ie.kind == ie.kind and

188

base_ie.reference_revision == ie.reference_revision):

189

ie.revision = base_ie.revision

190

ie.reference_revision = mapping.revision_id_foreign_to_bzr(hexsha)

191

texts.insert_record_stream([FulltextContentFactory((file_id, ie.revision), (), None, "")])

192

invdelta = [(oldpath, path, file_id, ie)]

193

return invdelta, {}, {}

174

raise SubmodulesNotSupported()

194

175

195

176

196

177

def remove_disappeared_children(path, base_children, existing_children):

205

186

return ret

206

187

207

188

208

def import_git_tree(texts, mapping, path, hexsha, base_inv, base_ie, parent_id,

209

revision_id, parent_invs, shagitmap, lookup_object, allow_submodules=False):

189

def import_git_tree(texts, mapping, path, hexsha, base_inv, base_ie, parent_id,

190

revision_id, parent_invs, shagitmap, lookup_object):

210

191

"""Import a git tree object into a bzr repository.

211

192

212

193

:param texts: VersionedFiles object to add to

218

199

invdelta = []

219

200

file_id = mapping.generate_file_id(path)

220

201

# We just have to hope this is indeed utf-8:

221

ie = InventoryDirectory(file_id, urlutils.basename(path.decode("utf-8")),

202

ie = InventoryDirectory(file_id, urlutils.basename(path.decode("utf-8")),

222

203

parent_id)

223

204

if base_ie is None:

224

205

# Newly appeared here

254

235

child_path = osutils.pathjoin(path, name)

255

236

if stat.S_ISDIR(mode):

256

237

subinvdelta, grandchildmodes, subshamap = import_git_tree(

257

texts, mapping, child_path, child_hexsha, base_inv,

258

base_children.get(basename), file_id, revision_id,

259

parent_invs, shagitmap, lookup_object,

260

allow_submodules=allow_submodules)

238

texts, mapping, child_path, child_hexsha, base_inv,

239

base_children.get(basename), file_id, revision_id, parent_invs, shagitmap,

240

lookup_object)

261

241

invdelta.extend(subinvdelta)

262

242

child_modes.update(grandchildmodes)

263

243

shamap.extend(subshamap)

264

244

elif S_ISGITLINK(mode): # submodule

265

if not allow_submodules:

266

raise SubmodulesRequireSubtrees()

267

245

subinvdelta, grandchildmodes, subshamap = import_git_submodule(

268

246

texts, mapping, child_path, child_hexsha, base_inv, base_children.get(basename),

269

247

file_id, revision_id, parent_invs, shagitmap, lookup_object)

271

249

child_modes.update(grandchildmodes)

272

250

shamap.extend(subshamap)

273

251

else:

274

subinvdelta, subshamap = import_git_blob(texts, mapping,

252

subinvdelta, subshamap = import_git_blob(texts, mapping,

275

253

child_path, child_hexsha, base_inv, base_children.get(basename), file_id,

276

revision_id, parent_invs, shagitmap, lookup_object,

254

revision_id, parent_invs, shagitmap, lookup_object,

277

255

mode_is_executable(mode), stat.S_ISLNK(mode))

278

256

invdelta.extend(subinvdelta)

279

257

shamap.extend(subshamap)

282

260

child_modes[child_path] = mode

283

261

# Remove any children that have disappeared

284

262

if base_ie is not None and base_ie.kind == "directory":

285

invdelta.extend(remove_disappeared_children(base_inv.id2path(file_id),

263

invdelta.extend(remove_disappeared_children(base_inv.id2path(file_id),

286

264

base_children, existing_children))

287

265

shamap.append((hexsha, "tree", (file_id, revision_id)))

288

266

return invdelta, child_modes, shamap

289

267

290

268

291

def approx_inv_size(inv):

292

# Very rough estimate, 1k per inventory entry

293

return len(inv) * 1024

294

295

296

def import_git_commit(repo, mapping, head, lookup_object,

297

target_git_object_retriever, parent_invs_cache):

298

o = lookup_object(head)

299

rev = mapping.import_commit(o)

300

# We have to do this here, since we have to walk the tree and

301

# we need to make sure to import the blobs / trees with the right

302

# path; this may involve adding them more than once.

303

parent_invs = []

304

for parent_id in rev.parent_ids:

305

try:

306

parent_invs.append(parent_invs_cache[parent_id])

307

except KeyError:

308

parent_inv = repo.get_inventory(parent_id)

309

parent_invs.append(parent_inv)

310

parent_invs_cache[parent_id] = parent_inv

311

if parent_invs == []:

312

base_inv = Inventory(root_id=None)

313

base_ie = None

314

else:

315

base_inv = parent_invs[0]

316

base_ie = base_inv.root

317

inv_delta, unusual_modes, shamap = import_git_tree(repo.texts,

318

mapping, "", o.tree, base_inv, base_ie, None, rev.revision_id,

319

parent_invs, target_git_object_retriever._idmap, lookup_object,

320

allow_submodules=getattr(repo._format, "supports_tree_reference", False))

321

target_git_object_retriever._idmap.add_entries(shamap)

322

if unusual_modes != {}:

323

for path, mode in unusual_modes.iteritems():

324

warn_unusual_mode(rev.foreign_revid, path, mode)

325

mapping.import_unusual_file_modes(rev, unusual_modes)

326

try:

327

basis_id = rev.parent_ids[0]

328

except IndexError:

329

basis_id = NULL_REVISION

330

base_inv = None

331

rev.inventory_sha1, inv = repo.add_inventory_by_delta(basis_id,

332

inv_delta, rev.revision_id, rev.parent_ids,

333

base_inv)

334

parent_invs_cache[rev.revision_id] = inv

335

repo.add_revision(rev.revision_id, rev)

336

if "verify" in debug.debug_flags:

337

new_unusual_modes = mapping.export_unusual_file_modes(rev)

338

if new_unusual_modes != unusual_modes:

339

raise AssertionError("unusual modes don't match: %r != %r" % (unusual_modes, new_unusual_modes))

340

objs = inventory_to_tree_and_blobs(inv, repo.texts, mapping, unusual_modes)

341

for sha1, newobj, path in objs:

342

assert path is not None

343

oldobj = tree_lookup_path(lookup_object, o.tree, path)

344

if oldobj != newobj:

345

raise AssertionError("%r != %r in %s" % (oldobj, newobj, path))

346

347

348

def import_git_objects(repo, mapping, object_iter, target_git_object_retriever,

269

def import_git_objects(repo, mapping, object_iter, target_git_object_retriever,

349

270

heads, pb=None):

350

271

"""Import a set of git objects into a bzr repository.

351

272

353

274

:param mapping: Mapping to use

354

275

:param object_iter: Iterator over Git objects.

355

276

"""

356

target_git_object_retriever._idmap.start_write_group() # FIXME: try/finally

357

277

def lookup_object(sha):

358

278

try:

359

279

return object_iter[sha]

361

281

return target_git_object_retriever[sha]

362

282

# TODO: a more (memory-)efficient implementation of this

363

283

graph = []

284

root_trees = {}

285

revisions = {}

364

286

checked = set()

365

287

heads = list(heads)

366

parent_invs_cache = lru_cache.LRUSizeCache(compute_size=approx_inv_size,

367

max_size=MAX_INV_CACHE_SIZE)

288

parent_invs_cache = LRUCache(50)

368

289

# Find and convert commit objects

369

290

while heads:

370

291

if pb is not None:

381

302

if repo.has_revision(rev.revision_id):

382

303

continue

383

304

squash_revision(repo, rev)

384

graph.append((o.id, o.parents))

385

target_git_object_retriever._idmap.add_entry(o.id, "commit",

305

root_trees[rev.revision_id] = o.tree

306

revisions[rev.revision_id] = rev

307

graph.append((rev.revision_id, rev.parent_ids))

308

target_git_object_retriever._idmap.add_entry(o.id, "commit",

386

309

(rev.revision_id, o.tree))

387

310

heads.extend([p for p in o.parents if p not in checked])

388

311

elif isinstance(o, Tag):

389

312

heads.append(o.object[1])

390

313

else:

391

314

trace.warning("Unable to import head object %r" % o)

392

checked.add(o.id)

393

del checked

315

checked.add(head)

394

316

# Order the revisions

395

317

# Create the inventory objects

396

batch_size = 100

397

revision_ids = topo_sort(graph)

398

pack_hints = []

399

for offset in range(0, len(revision_ids), batch_size):

400

repo.start_write_group()

318

for i, revid in enumerate(topo_sort(graph)):

319

if pb is not None:

320

pb.update("fetching revisions", i, len(graph))

321

rev = revisions[revid]

322

# We have to do this here, since we have to walk the tree and

323

# we need to make sure to import the blobs / trees with the right

324

# path; this may involve adding them more than once.

325

parent_invs = []

326

for parent_id in rev.parent_ids:

327

try:

328

parent_invs.append(parent_invs_cache[parent_id])

329

except KeyError:

330

parent_inv = repo.get_inventory(parent_id)

331

parent_invs.append(parent_inv)

332

parent_invs_cache[parent_id] = parent_inv

333

if parent_invs == []:

334

base_inv = Inventory(root_id=None)

335

base_ie = None

336

else:

337

base_inv = parent_invs[0]

338

base_ie = base_inv.root

339

inv_delta, unusual_modes, shamap = import_git_tree(repo.texts,

340

mapping, "", root_trees[revid], base_inv, base_ie, None, revid,

341

parent_invs, target_git_object_retriever._idmap, lookup_object)

342

target_git_object_retriever._idmap.add_entries(shamap)

343

if unusual_modes != {}:

344

for path, mode in unusual_modes.iteritems():

345

warn_unusual_mode(rev.foreign_revid, path, mode)

346

mapping.import_unusual_file_modes(rev, unusual_modes)

401

347

try:

402

for i, head in enumerate(revision_ids[offset:offset+batch_size]):

403

if pb is not None:

404

pb.update("fetching revisions", offset+i, len(revision_ids))

405

import_git_commit(repo, mapping, head, lookup_object,

406

target_git_object_retriever,

407

parent_invs_cache)

408

except:

409

repo.abort_write_group()

410

raise

411

else:

412

hint = repo.commit_write_group()

413

if hint is not None:

414

pack_hints.extend(hint)

415

target_git_object_retriever._idmap.commit_write_group()

416

return pack_hints

348

basis_id = rev.parent_ids[0]

349

except IndexError:

350

basis_id = NULL_REVISION

351

rev.inventory_sha1, inv = repo.add_inventory_by_delta(basis_id,

352

inv_delta, rev.revision_id, rev.parent_ids)

353

parent_invs_cache[rev.revision_id] = inv

354

repo.add_revision(rev.revision_id, rev)

355

if "verify" in debug.debug_flags:

356

new_unusual_modes = mapping.export_unusual_file_modes(rev)

357

if new_unusual_modes != unusual_modes:

358

raise AssertionError("unusual modes don't match: %r != %r" % (unusual_modes, new_unusual_modes))

359

objs = inventory_to_tree_and_blobs(inv, repo.texts, mapping, unusual_modes)

360

for sha1, newobj, path in objs:

361

assert path is not None

362

oldobj = tree_lookup_path(lookup_object, root_trees[revid], path)

363

if oldobj != newobj:

364

raise AssertionError("%r != %r in %s" % (oldobj, newobj, path))

365

366

target_git_object_retriever._idmap.commit()

417

367

418

368

419

369

class InterGitRepository(InterRepository):

435

385

436

386

437

387

class InterGitNonGitRepository(InterGitRepository):

438

"""Base InterRepository that copies revisions from a Git into a non-Git

388

"""Base InterRepository that copies revisions from a Git into a non-Git

439

389

repository."""

440

390

441

def fetch_refs(self, revision_id=None, pb=None, find_ghosts=False,

391

def fetch_refs(self, revision_id=None, pb=None, find_ghosts=False,

442

392

mapping=None, fetch_spec=None):

443

393

if mapping is None:

444

394

mapping = self.source.get_mapping()

479

429

480

430

481

431

class InterRemoteGitNonGitRepository(InterGitNonGitRepository):

482

"""InterRepository that copies revisions from a remote Git into a non-Git

432

"""InterRepository that copies revisions from a remote Git into a non-Git

483

433

repository."""

484

434

485

435

def get_target_heads(self):

505

455

wants = determine_wants(heads)

506

456

recorded_wants.extend(wants)

507

457

return wants

508

458

509

459

create_pb = None

510

460

if pb is None:

511

461

create_pb = pb = ui.ui_factory.nested_progress_bar()

512

462

try:

513

objects_iter = self.source.fetch_objects(

514

record_determine_wants, graph_walker,

515

store.get_raw, progress)

516

return import_git_objects(self.target, mapping,

517

objects_iter, store, recorded_wants, pb)

463

self.target.start_write_group()

464

try:

465

objects_iter = self.source.fetch_objects(

466

record_determine_wants, graph_walker,

467

store.get_raw, progress)

468

import_git_objects(self.target, mapping, objects_iter,

469

store, recorded_wants, pb)

470

finally:

471

pack_hint = self.target.commit_write_group()

472

return pack_hint

518

473

finally:

519

474

if create_pb:

520

475

create_pb.finished()

525

480

def is_compatible(source, target):

526

481

"""Be compatible with GitRepository."""

527

482

# FIXME: Also check target uses VersionedFile

528

return (isinstance(source, RemoteGitRepository) and

483

return (isinstance(source, RemoteGitRepository) and

529

484

target.supports_rich_root() and

530

485

not isinstance(target, GitRepository))

531

486

532

487

533

488

class InterLocalGitNonGitRepository(InterGitNonGitRepository):

534

"""InterRepository that copies revisions from a local Git into a non-Git

489

"""InterRepository that copies revisions from a local Git into a non-Git

535

490

repository."""

536

491

537

492

def fetch_objects(self, determine_wants, mapping, pb=None):

543

498

try:

544

499

self.target.lock_write()

545

500

try:

546

return import_git_objects(self.target, mapping,

547

self.source._git.object_store, target_git_object_retriever,

548

wants, pb)

501

self.target.start_write_group()

502

try:

503

import_git_objects(self.target, mapping,

504

self.source._git.object_store,

505

target_git_object_retriever, wants, pb)

506

finally:

507

pack_hint = self.target.commit_write_group()

508

return pack_hint

549

509

finally:

550

510

self.target.unlock()

551

511

finally:

556

516

def is_compatible(source, target):

557

517

"""Be compatible with GitRepository."""

558

518

# FIXME: Also check target uses VersionedFile

559

return (isinstance(source, LocalGitRepository) and

519

return (isinstance(source, LocalGitRepository) and

560

520

target.supports_rich_root() and

561

521

not isinstance(target, GitRepository))

562

522

564

524

class InterGitGitRepository(InterGitRepository):

565

525

"""InterRepository that copies between Git repositories."""

566

526

567

def fetch_objects(self, determine_wants, mapping, pb=None):

568

def progress(text):

569

trace.note("git: %s", text)

570

graphwalker = self.target._git.get_graph_walker()

571

if (isinstance(self.source, LocalGitRepository) and

572

isinstance(self.target, LocalGitRepository)):

573

return self.source._git.fetch(self.target._git, determine_wants,

574

progress)

575

elif (isinstance(self.source, LocalGitRepository) and

576

isinstance(self.target, RemoteGitRepository)):

577

raise NotImplementedError

578

elif (isinstance(self.source, RemoteGitRepository) and

579

isinstance(self.target, LocalGitRepository)):

580

f, commit = self.target._git.object_store.add_thin_pack()

581

try:

582

refs = self.source._git.fetch_pack(determine_wants,

583

graphwalker, f.write, progress)

584

commit()

585

return refs

586

except:

587

f.close()

588

raise

589

else:

590

raise AssertionError

591

592

def fetch_refs(self, revision_id=None, pb=None, find_ghosts=False,

527

def fetch_refs(self, revision_id=None, pb=None, find_ghosts=False,

593

528

mapping=None, fetch_spec=None, branches=None):

594

529

if mapping is None:

595

530

mapping = self.source.get_mapping()

531

def progress(text):

532

trace.info("git: %s", text)

596

533

r = self.target._git

597

534

if revision_id is not None:

598

535

args = [mapping.revision_id_bzr_to_foreign(revision_id)[0]]

604

541

determine_wants = r.object_store.determine_wants_all

605

542

else:

606

543

determine_wants = lambda x: [y for y in args if not y in r.object_store]

607

return self.fetch_objects(determine_wants, mapping)

608

544

545

graphwalker = r.get_graph_walker()

546

f, commit = r.object_store.add_thin_pack()

547

try:

548

refs = self.source.fetch_pack(determine_wants, graphwalker,

549

f.write, progress)

550

commit()

551

return refs

552

except:

553

f.close()

554

raise

609

555

610

556

@staticmethod

611

557

def is_compatible(source, target):

612

558

"""Be compatible with GitRepository."""

613

return (isinstance(source, GitRepository) and

559

return (isinstance(source, GitRepository) and

614

560

isinstance(target, GitRepository))