/brz/remove-bazaar : revision 0.200.672

14

# along with this program; if not, write to the Free Software

15

# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA

16

17

from cStringIO import (

18

StringIO,

19

)

20

import dulwich as git

21

17

from dulwich.objects import (

22

18

Commit,

23

19

Tag,

26

22

from dulwich.object_store import (

27

23

tree_lookup_path,

28

24

)

25

import re

29

26

import stat

30

27

31

28

from bzrlib import (

32

29

debug,

30

lru_cache,

33

31

osutils,

34

32

trace,

35

33

ui,

36

34

urlutils,

37

35

)

38

36

from bzrlib.errors import (

39

InvalidRevisionId,

37

BzrError,

40

38

NoSuchId,

41

NoSuchRevision,

42

39

)

43

40

from bzrlib.inventory import (

44

41

Inventory,

45

42

InventoryDirectory,

46

43

InventoryFile,

47

44

InventoryLink,

48

)

49

from bzrlib.lru_cache import (

50

LRUCache,

45

TreeReference,

51

46

)

52

47

from bzrlib.repository import (

53

48

InterRepository,

67

62

inventory_to_tree_and_blobs,

68

63

mode_is_executable,

69

64

squash_revision,

70

text_to_blob,

71

65

warn_unusual_mode,

72

66

)

73

67

from bzrlib.plugins.git.object_store import (

77

71

RemoteGitRepository,

78

72

)

79

73

from bzrlib.plugins.git.repository import (

80

GitRepository,

74

GitRepository,

81

75

GitRepositoryFormat,

82

76

LocalGitRepository,

83

77

)

84

78

85

79

86

def import_git_blob(texts, mapping, path, hexsha, base_inv, parent_id,

80

MAX_INV_CACHE_SIZE = 50 * 1024 * 1024

81

82

83

def import_git_blob(texts, mapping, path, hexsha, base_inv, base_ie, parent_id,

87

84

revision_id, parent_invs, shagitmap, lookup_object, executable, symlink):

88

85

"""Import a git blob object into a bzr repository.

89

86

101

98

ie = cls(file_id, urlutils.basename(path).decode("utf-8"), parent_id)

102

99

ie.executable = executable

103

100

# See if this has changed at all

104

try:

105

base_ie = base_inv[file_id]

106

except NoSuchId:

107

base_ie = None

101

if base_ie is None:

108

102

base_sha = None

109

103

else:

110

104

try:

160

154

shamap = [(hexsha, "blob", (ie.file_id, ie.revision))]

161

155

else:

162

156

shamap = []

163

if file_id in base_inv:

157

invdelta = []

158

if base_ie is not None:

164

159

old_path = base_inv.id2path(file_id)

160

if base_ie.kind == "directory":

161

invdelta.extend(remove_disappeared_children(old_path, base_ie.children, []))

165

162

else:

166

163

old_path = None

167

invdelta = [(old_path, path, file_id, ie)]

168

invdelta.extend(remove_disappeared_children(base_inv, base_ie, []))

164

invdelta.append((old_path, path, file_id, ie))

169

165

return (invdelta, shamap)

170

166

171

167

172

def import_git_submodule(texts, mapping, path, hexsha, base_inv, parent_id,

173

revision_id, parent_invs, shagitmap, lookup_object):

174

raise NotImplementedError(import_git_submodule)

175

176

177

def remove_disappeared_children(base_inv, base_ie, existing_children):

178

if base_ie is None or base_ie.kind != 'directory':

179

return []

168

class SubmodulesRequireSubtrees(BzrError):

169

_fmt = """The repository you are fetching from contains submodules. To continue, upgrade your Bazaar repository to a format that supports nested trees, such as 'development-subtree'."""

170

internal = False

171

172

173

def import_git_submodule(texts, mapping, path, hexsha, base_inv, base_ie,

174

parent_id, revision_id, parent_invs, shagitmap, lookup_object):

175

file_id = mapping.generate_file_id(path)

176

ie = TreeReference(file_id, urlutils.basename(path.decode("utf-8")),

177

parent_id)

178

ie.revision = revision_id

179

if base_ie is None:

180

oldpath = None

181

else:

182

oldpath = path

183

if base_ie.kind == ie.kind and base_ie.reference_revision == ie.reference_revision:

184

ie.revision = base_ie.revision

185

ie.reference_revision = mapping.revision_id_foreign_to_bzr(hexsha)

186

texts.insert_record_stream([FulltextContentFactory((file_id, ie.revision), (), None, "")])

187

invdelta = [(oldpath, path, file_id, ie)]

188

return invdelta, {}, {}

189

190

191

def remove_disappeared_children(path, base_children, existing_children):

180

192

ret = []

181

deletable = [v for k,v in base_ie.children.iteritems() if k not in existing_children]

193

deletable = [(osutils.pathjoin(path, k), v) for k,v in base_children.iteritems() if k not in existing_children]

182

194

while deletable:

183

ie = deletable.pop()

184

ret.append((base_inv.id2path(ie.file_id), None, ie.file_id, None))

195

(path, ie) = deletable.pop()

196

ret.append((path, None, ie.file_id, None))

185

197

if ie.kind == "directory":

186

deletable.extend(ie.children.values())

198

for name, child_ie in ie.children.iteritems():

199

deletable.append((osutils.pathjoin(path, name), child_ie))

187

200

return ret

188

201

189

202

190

def import_git_tree(texts, mapping, path, hexsha, base_inv, parent_id,

191

revision_id, parent_invs, shagitmap, lookup_object):

203

def import_git_tree(texts, mapping, path, hexsha, base_inv, base_ie, parent_id,

204

revision_id, parent_invs, shagitmap, lookup_object, allow_submodules=False):

192

205

"""Import a git tree object into a bzr repository.

193

206

194

207

:param texts: VersionedFiles object to add to

200

213

invdelta = []

201

214

file_id = mapping.generate_file_id(path)

202

215

# We just have to hope this is indeed utf-8:

203

ie = InventoryDirectory(file_id, urlutils.basename(path.decode("utf-8")),

216

ie = InventoryDirectory(file_id, urlutils.basename(path.decode("utf-8")),

204

217

parent_id)

205

try:

206

base_ie = base_inv[file_id]

207

except NoSuchId:

218

if base_ie is None:

208

219

# Newly appeared here

209

base_ie = None

210

220

ie.revision = revision_id

211

texts.add_lines((file_id, ie.revision), (), [])

221

texts.insert_record_stream([FulltextContentFactory((file_id, ie.revision), (), None, "")])

212

222

invdelta.append((None, path, file_id, ie))

213

223

else:

214

224

# See if this has changed at all

222

232

return [], {}, []

223

233

if base_ie.kind != "directory":

224

234

ie.revision = revision_id

225

texts.add_lines((ie.file_id, ie.revision), (), [])

235

texts.insert_record_stream([FulltextContentFactory((ie.file_id, ie.revision), (), None, "")])

226

236

invdelta.append((base_inv.id2path(ie.file_id), path, ie.file_id, ie))

237

if base_ie is not None and base_ie.kind == "directory":

238

base_children = base_ie.children

239

else:

240

base_children = {}

227

241

# Remember for next time

228

242

existing_children = set()

229

243

child_modes = {}

235

249

child_path = osutils.pathjoin(path, name)

236

250

if stat.S_ISDIR(mode):

237

251

subinvdelta, grandchildmodes, subshamap = import_git_tree(

238

texts, mapping, child_path, child_hexsha, base_inv,

239

file_id, revision_id, parent_invs, shagitmap, lookup_object)

252

texts, mapping, child_path, child_hexsha, base_inv,

253

base_children.get(basename), file_id, revision_id,

254

parent_invs, shagitmap, lookup_object,

255

allow_submodules=allow_submodules)

240

256

invdelta.extend(subinvdelta)

241

257

child_modes.update(grandchildmodes)

242

258

shamap.extend(subshamap)

243

259

elif S_ISGITLINK(mode): # submodule

260

if not allow_submodules:

261

raise SubmodulesRequireSubtrees()

244

262

subinvdelta, grandchildmodes, subshamap = import_git_submodule(

245

texts, mapping, child_path, child_hexsha, base_inv,

263

texts, mapping, child_path, child_hexsha, base_inv, base_children.get(basename),

246

264

file_id, revision_id, parent_invs, shagitmap, lookup_object)

247

265

invdelta.extend(subinvdelta)

248

266

child_modes.update(grandchildmodes)

249

267

shamap.extend(subshamap)

250

268

else:

251

subinvdelta, subshamap = import_git_blob(texts, mapping,

252

child_path, child_hexsha, base_inv, file_id, revision_id,

253

parent_invs, shagitmap, lookup_object,

269

subinvdelta, subshamap = import_git_blob(texts, mapping,

270

child_path, child_hexsha, base_inv, base_children.get(basename), file_id,

271

revision_id, parent_invs, shagitmap, lookup_object,

254

272

mode_is_executable(mode), stat.S_ISLNK(mode))

255

273

invdelta.extend(subinvdelta)

256

274

shamap.extend(subshamap)

258

276

stat.S_IFLNK, DEFAULT_FILE_MODE|0111):

259

277

child_modes[child_path] = mode

260

278

# Remove any children that have disappeared

261

invdelta.extend(remove_disappeared_children(base_inv, base_ie, existing_children))

279

if base_ie is not None and base_ie.kind == "directory":

280

invdelta.extend(remove_disappeared_children(base_inv.id2path(file_id),

281

base_children, existing_children))

262

282

shamap.append((hexsha, "tree", (file_id, revision_id)))

263

283

return invdelta, child_modes, shamap

264

284

265

285

266

def import_git_objects(repo, mapping, object_iter, target_git_object_retriever,

286

def approx_inv_size(inv):

287

# Very rough estimate, 1k per inventory entry

288

return len(inv) * 1024

289

290

291

def import_git_objects(repo, mapping, object_iter, target_git_object_retriever,

267

292

heads, pb=None):

268

293

"""Import a set of git objects into a bzr repository.

269

294

278

303

return target_git_object_retriever[sha]

279

304

# TODO: a more (memory-)efficient implementation of this

280

305

graph = []

281

root_trees = {}

282

revisions = {}

283

306

checked = set()

284

307

heads = list(heads)

285

parent_invs_cache = LRUCache(50)

308

parent_invs_cache = lru_cache.LRUSizeCache(compute_size=approx_inv_size,

309

max_size=MAX_INV_CACHE_SIZE)

286

310

# Find and convert commit objects

287

311

while heads:

288

312

if pb is not None:

292

316

try:

293

317

o = lookup_object(head)

294

318

except KeyError:

319

trace.mutter('missing head %s', head)

295

320

continue

296

321

if isinstance(o, Commit):

297

322

rev = mapping.import_commit(o)

298

323

if repo.has_revision(rev.revision_id):

299

324

continue

300

325

squash_revision(repo, rev)

301

root_trees[rev.revision_id] = o.tree

302

revisions[rev.revision_id] = rev

303

graph.append((rev.revision_id, rev.parent_ids))

304

target_git_object_retriever._idmap.add_entry(o.id, "commit",

326

graph.append((o.id, o.parents))

327

target_git_object_retriever._idmap.add_entry(o.id, "commit",

305

328

(rev.revision_id, o.tree))

306

329

heads.extend([p for p in o.parents if p not in checked])

307

330

elif isinstance(o, Tag):

308

331

heads.append(o.object[1])

309

332

else:

310

333

trace.warning("Unable to import head object %r" % o)

311

checked.add(head)

334

checked.add(o.id)

335

del checked

312

336

# Order the revisions

313

337

# Create the inventory objects

314

for i, revid in enumerate(topo_sort(graph)):

338

for i, head in enumerate(topo_sort(graph)):

315

339

if pb is not None:

316

340

pb.update("fetching revisions", i, len(graph))

317

rev = revisions[revid]

318

# We have to do this here, since we have to walk the tree and

319

# we need to make sure to import the blobs / trees with the right

341

o = lookup_object(head)

342

rev = mapping.import_commit(o)

343

# We have to do this here, since we have to walk the tree and

344

# we need to make sure to import the blobs / trees with the right

320

345

# path; this may involve adding them more than once.

321

346

parent_invs = []

322

347

for parent_id in rev.parent_ids:

328

353

parent_invs_cache[parent_id] = parent_inv

329

354

if parent_invs == []:

330

355

base_inv = Inventory(root_id=None)

356

base_ie = None

331

357

else:

332

358

base_inv = parent_invs[0]

333

inv_delta, unusual_modes, shamap = import_git_tree(repo.texts,

334

mapping, "", root_trees[revid], base_inv, None, revid,

335

parent_invs, target_git_object_retriever._idmap, lookup_object)

359

base_ie = base_inv.root

360

inv_delta, unusual_modes, shamap = import_git_tree(repo.texts,

361

mapping, "", o.tree, base_inv, base_ie, None, rev.revision_id,

362

parent_invs, target_git_object_retriever._idmap, lookup_object,

363

allow_submodules=getattr(repo._format, "supports_tree_reference", False))

336

364

target_git_object_retriever._idmap.add_entries(shamap)

337

365

if unusual_modes != {}:

338

366

for path, mode in unusual_modes.iteritems():

342

370

basis_id = rev.parent_ids[0]

343

371

except IndexError:

344

372

basis_id = NULL_REVISION

373

base_inv = None

345

374

rev.inventory_sha1, inv = repo.add_inventory_by_delta(basis_id,

346

inv_delta, rev.revision_id, rev.parent_ids)

375

inv_delta, rev.revision_id, rev.parent_ids,

376

base_inv)

347

377

parent_invs_cache[rev.revision_id] = inv

348

378

repo.add_revision(rev.revision_id, rev)

349

379

if "verify" in debug.debug_flags:

353

383

objs = inventory_to_tree_and_blobs(inv, repo.texts, mapping, unusual_modes)

354

384

for sha1, newobj, path in objs:

355

385

assert path is not None

356

oldobj = tree_lookup_path(lookup_object, root_trees[revid], path)

386

oldobj = tree_lookup_path(lookup_object, o.tree, path)

357

387

if oldobj != newobj:

358

388

raise AssertionError("%r != %r in %s" % (oldobj, newobj, path))

359

389

379

409

380

410

381

411

class InterGitNonGitRepository(InterGitRepository):

382

"""Base InterRepository that copies revisions from a Git into a non-Git

412

"""Base InterRepository that copies revisions from a Git into a non-Git

383

413

repository."""

384

414

385

def fetch_refs(self, revision_id=None, pb=None, find_ghosts=False,

415

def fetch_refs(self, revision_id=None, pb=None, find_ghosts=False,

386

416

mapping=None, fetch_spec=None):

387

417

if mapping is None:

388

418

mapping = self.source.get_mapping()

400

430

else:

401

431

ret = [mapping.revision_id_bzr_to_foreign(revid)[0] for revid in interesting_heads if revid not in (None, NULL_REVISION)]

402

432

return [rev for rev in ret if not self.target.has_revision(mapping.revision_id_foreign_to_bzr(rev))]

403

self.fetch_objects(determine_wants, mapping, pb)

433

pack_hint = self.fetch_objects(determine_wants, mapping, pb)

434

if pack_hint is not None and self.target._format.pack_compresses:

435

self.target.pack(hint=pack_hint)

436

if interesting_heads is not None:

437

present_interesting_heads = self.target.has_revisions(interesting_heads)

438

missing_interesting_heads = set(interesting_heads) - present_interesting_heads

439

if missing_interesting_heads:

440

raise AssertionError("Missing interesting heads: %r" % missing_interesting_heads)

404

441

return self._refs

405

442

406

443

444

_GIT_PROGRESS_RE = re.compile(r"(.*?): +(\d+)% \((\d+)/(\d+)\)")

445

def report_git_progress(pb, text):

446

text = text.rstrip("\r\n")

447

g = _GIT_PROGRESS_RE.match(text)

448

if g is not None:

449

(text, pct, current, total) = g.groups()

450

pb.update(text, int(current), int(total))

451

else:

452

pb.update(text, 0, 0)

453

454

407

455

class InterRemoteGitNonGitRepository(InterGitNonGitRepository):

408

"""InterRepository that copies revisions from a remote Git into a non-Git

456

"""InterRepository that copies revisions from a remote Git into a non-Git

409

457

repository."""

410

458

459

def get_target_heads(self):

460

# FIXME: This should be more efficient

461

all_revs = self.target.all_revision_ids()

462

parent_map = self.target.get_parent_map(all_revs)

463

all_parents = set()

464

map(all_parents.update, parent_map.itervalues())

465

return set(all_revs) - all_parents

466

411

467

def fetch_objects(self, determine_wants, mapping, pb=None):

412

468

def progress(text):

413

pb.update("git: %s" % text.rstrip("\r\n"), 0, 0)

469

report_git_progress(pb, text)

414

470

store = BazaarObjectStore(self.target, mapping)

415

471

self.target.lock_write()

416

472

try:

417

heads = self.target.get_graph().heads(self.target.all_revision_ids())

473

heads = self.get_target_heads()

418

474

graph_walker = store.get_graph_walker(

419

475

[store._lookup_revision_sha1(head) for head in heads])

420

476

recorded_wants = []

423

479

wants = determine_wants(heads)

424

480

recorded_wants.extend(wants)

425

481

return wants

426

482

427

483

create_pb = None

428

484

if pb is None:

429

485

create_pb = pb = ui.ui_factory.nested_progress_bar()

431

487

self.target.start_write_group()

432

488

try:

433

489

objects_iter = self.source.fetch_objects(

434

record_determine_wants, graph_walker,

490

record_determine_wants, graph_walker,

435

491

store.get_raw, progress)

436

import_git_objects(self.target, mapping, objects_iter,

492

import_git_objects(self.target, mapping, objects_iter,

437

493

store, recorded_wants, pb)

438

494

finally:

439

self.target.commit_write_group()

495

pack_hint = self.target.commit_write_group()

496

return pack_hint

440

497

finally:

441

498

if create_pb:

442

499

create_pb.finished()

447

504

def is_compatible(source, target):

448

505

"""Be compatible with GitRepository."""

449

506

# FIXME: Also check target uses VersionedFile

450

return (isinstance(source, RemoteGitRepository) and

507

return (isinstance(source, RemoteGitRepository) and

451

508

target.supports_rich_root() and

452

509

not isinstance(target, GitRepository))

453

510

454

511

455

512

class InterLocalGitNonGitRepository(InterGitNonGitRepository):

456

"""InterRepository that copies revisions from a local Git into a non-Git

513

"""InterRepository that copies revisions from a local Git into a non-Git

457

514

repository."""

458

515

459

516

def fetch_objects(self, determine_wants, mapping, pb=None):

467

524

try:

468

525

self.target.start_write_group()

469

526

try:

470

import_git_objects(self.target, mapping,

471

self.source._git.object_store,

527

import_git_objects(self.target, mapping,

528

self.source._git.object_store,

472

529

target_git_object_retriever, wants, pb)

473

530

finally:

474

self.target.commit_write_group()

531

pack_hint = self.target.commit_write_group()

532

return pack_hint

475

533

finally:

476

534

self.target.unlock()

477

535

finally:

482

540

def is_compatible(source, target):

483

541

"""Be compatible with GitRepository."""

484

542

# FIXME: Also check target uses VersionedFile

485

return (isinstance(source, LocalGitRepository) and

543

return (isinstance(source, LocalGitRepository) and

486

544

target.supports_rich_root() and

487

545

not isinstance(target, GitRepository))

488

546

490

548

class InterGitGitRepository(InterGitRepository):

491

549

"""InterRepository that copies between Git repositories."""

492

550

493

def fetch_refs(self, revision_id=None, pb=None, find_ghosts=False,

551

def fetch_objects(self, determine_wants, mapping, pb=None):

552

def progress(text):

553

trace.note("git: %s", text)

554

graphwalker = self.target._git.get_graph_walker()

555

if isinstance(self.source, LocalGitRepository) and isinstance(self.target, LocalGitRepository):

556

return self.source._git.fetch(self.target._git, determine_wants,

557

progress)

558

elif isinstance(self.source, LocalGitRepository) and isinstance(self.target, RemoteGitRepository):

559

raise NotImplementedError

560

elif isinstance(self.source, RemoteGitRepository) and isinstance(self.target, LocalGitRepository):

561

f, commit = self.target._git.object_store.add_thin_pack()

562

try:

563

refs = self.source._git.fetch_pack(determine_wants, graphwalker,

564

f.write, progress)

565

commit()

566

return refs

567

except:

568

f.close()

569

raise

570

else:

571

raise AssertionError

572

573

def fetch_refs(self, revision_id=None, pb=None, find_ghosts=False,

494

574

mapping=None, fetch_spec=None, branches=None):

495

575

if mapping is None:

496

576

mapping = self.source.get_mapping()

497

def progress(text):

498

trace.info("git: %s", text)

499

577

r = self.target._git

500

578

if revision_id is not None:

501

579

args = [mapping.revision_id_bzr_to_foreign(revision_id)[0]]

507

585

determine_wants = r.object_store.determine_wants_all

508

586

else:

509

587

determine_wants = lambda x: [y for y in args if not y in r.object_store]

588

return self.fetch_objects(determine_wants, mapping)

510

589

511

graphwalker = r.get_graph_walker()

512

f, commit = r.object_store.add_thin_pack()

513

try:

514

refs = self.source.fetch_pack(determine_wants, graphwalker,

515

f.write, progress)

516

commit()

517

return refs

518

except:

519

f.close()

520

raise

521

590

522

591

@staticmethod

523

592

def is_compatible(source, target):

524

593

"""Be compatible with GitRepository."""

525

return (isinstance(source, GitRepository) and

594

return (isinstance(source, GitRepository) and

526

595

isinstance(target, GitRepository))