/brz/remove-bazaar : revision 0.200.1

1

2

#

3

# This program is free software; you can redistribute it and/or modify

4

# it under the terms of the GNU General Public License as published by

5

# the Free Software Foundation; either version 2 of the License, or

6

# (at your option) any later version.

7

#

8

# This program is distributed in the hope that it will be useful,

9

# but WITHOUT ANY WARRANTY; without even the implied warranty of

10

# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the

11

# GNU General Public License for more details.

12

#

13

# You should have received a copy of the GNU General Public License

14

# along with this program; if not, write to the Free Software

15

# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA

16

17

from cStringIO import (

18

StringIO,

19

)

20

import dulwich as git

21

from dulwich.objects import (

22

Commit,

23

Tag,

24

S_ISGITLINK,

25

)

26

from dulwich.object_store import (

27

tree_lookup_path,

28

)

29

import stat

30

31

from bzrlib import (

32

debug,

33

osutils,

34

trace,

35

ui,

36

urlutils,

37

)

38

from bzrlib.errors import (

39

InvalidRevisionId,

40

NoSuchId,

41

NoSuchRevision,

42

)

43

from bzrlib.inventory import (

44

Inventory,

45

InventoryDirectory,

46

InventoryFile,

47

InventoryLink,

48

)

49

from bzrlib.lru_cache import (

50

LRUCache,

51

)

52

from bzrlib.repository import (

53

InterRepository,

54

)

55

from bzrlib.revision import (

56

NULL_REVISION,

57

)

58

from bzrlib.tsort import (

59

topo_sort,

60

)

61

from bzrlib.versionedfile import (

62

FulltextContentFactory,

63

)

64

65

from bzrlib.plugins.git.mapping import (

66

DEFAULT_FILE_MODE,

67

inventory_to_tree_and_blobs,

68

mode_is_executable,

69

squash_revision,

70

text_to_blob,

71

warn_unusual_mode,

72

)

73

from bzrlib.plugins.git.object_store import (

74

BazaarObjectStore,

75

)

76

from bzrlib.plugins.git.remote import (

77

RemoteGitRepository,

78

)

79

from bzrlib.plugins.git.repository import (

80

GitRepository,

81

GitRepositoryFormat,

82

LocalGitRepository,

83

)

84

85

86

def import_git_blob(texts, mapping, path, hexsha, base_inv, parent_id,

87

revision_id, parent_invs, shagitmap, lookup_object, executable, symlink):

88

"""Import a git blob object into a bzr repository.

89

90

:param texts: VersionedFiles to add to

91

:param path: Path in the tree

92

:param blob: A git blob

93

:return: Inventory delta for this file

94

"""

95

file_id = mapping.generate_file_id(path)

96

if symlink:

97

cls = InventoryLink

98

else:

99

cls = InventoryFile

100

# We just have to hope this is indeed utf-8:

101

ie = cls(file_id, urlutils.basename(path).decode("utf-8"), parent_id)

102

ie.executable = executable

103

# See if this has changed at all

104

try:

105

base_ie = base_inv[file_id]

106

except NoSuchId:

107

base_ie = None

108

base_sha = None

109

else:

110

try:

111

base_sha = shagitmap.lookup_blob(file_id, base_ie.revision)

112

except KeyError:

113

base_sha = None

114

else:

115

if (base_sha == hexsha and base_ie.executable == ie.executable

116

and base_ie.kind == ie.kind):

117

# If nothing has changed since the base revision, we're done

118

return [], []

119

if base_sha == hexsha and base_ie.kind == ie.kind:

120

ie.text_size = base_ie.text_size

121

ie.text_sha1 = base_ie.text_sha1

122

ie.symlink_target = base_ie.symlink_target

123

if ie.executable == base_ie.executable:

124

ie.revision = base_ie.revision

125

else:

126

blob = lookup_object(hexsha)

127

else:

128

blob = lookup_object(hexsha)

129

if ie.kind == "symlink":

130

ie.revision = None

131

ie.symlink_target = blob.data

132

ie.text_size = None

133

ie.text_sha1 = None

134

else:

135

ie.text_size = len(blob.data)

136

ie.text_sha1 = osutils.sha_string(blob.data)

137

# Check what revision we should store

138

parent_keys = []

139

for pinv in parent_invs:

140

if pinv.revision_id == base_inv.revision_id:

141

pie = base_ie

142

if pie is None:

143

continue

144

else:

145

try:

146

pie = pinv[file_id]

147

except NoSuchId:

148

continue

149

if pie.text_sha1 == ie.text_sha1 and pie.executable == ie.executable and pie.symlink_target == ie.symlink_target:

150

# found a revision in one of the parents to use

151

ie.revision = pie.revision

152

break

153

parent_keys.append((file_id, pie.revision))

154

if ie.revision is None:

155

# Need to store a new revision

156

ie.revision = revision_id

157

assert file_id is not None

158

assert ie.revision is not None

159

texts.insert_record_stream([FulltextContentFactory((file_id, ie.revision), tuple(parent_keys), ie.text_sha1, blob.data)])

160

shamap = [(hexsha, "blob", (ie.file_id, ie.revision))]

161

else:

162

shamap = []

163

if file_id in base_inv:

164

old_path = base_inv.id2path(file_id)

165

else:

166

old_path = None

167

invdelta = [(old_path, path, file_id, ie)]

168

invdelta.extend(remove_disappeared_children(base_inv, base_ie, []))

169

return (invdelta, shamap)

170

171

172

def import_git_submodule(texts, mapping, path, hexsha, base_inv, parent_id,

173

revision_id, parent_invs, shagitmap, lookup_object):

174

raise NotImplementedError(import_git_submodule)

175

176

177

def remove_disappeared_children(base_inv, base_ie, existing_children):

178

if base_ie is None or base_ie.kind != 'directory':

179

return []

180

ret = []

181

deletable = [v for k,v in base_ie.children.iteritems() if k not in existing_children]

182

while deletable:

183

ie = deletable.pop()

184

ret.append((base_inv.id2path(ie.file_id), None, ie.file_id, None))

185

if ie.kind == "directory":

186

deletable.extend(ie.children.values())

187

return ret

188

189

190

def import_git_tree(texts, mapping, path, hexsha, base_inv, parent_id,

191

revision_id, parent_invs, shagitmap, lookup_object):

192

"""Import a git tree object into a bzr repository.

193

194

:param texts: VersionedFiles object to add to

195

:param path: Path in the tree

196

:param tree: A git tree object

197

:param base_inv: Base inventory against which to return inventory delta

198

:return: Inventory delta for this subtree

199

"""

200

invdelta = []

201

file_id = mapping.generate_file_id(path)

202

# We just have to hope this is indeed utf-8:

203

ie = InventoryDirectory(file_id, urlutils.basename(path.decode("utf-8")),

204

parent_id)

205

try:

206

base_ie = base_inv[file_id]

207

except NoSuchId:

208

# Newly appeared here

209

base_ie = None

210

ie.revision = revision_id

211

texts.add_lines((file_id, ie.revision), (), [])

212

invdelta.append((None, path, file_id, ie))

213

else:

214

# See if this has changed at all

215

try:

216

base_sha = shagitmap.lookup_tree(file_id, base_inv.revision_id)

217

except KeyError:

218

pass

219

else:

220

if base_sha == hexsha:

221

# If nothing has changed since the base revision, we're done

222

return [], {}, []

223

if base_ie.kind != "directory":

224

ie.revision = revision_id

225

texts.add_lines((ie.file_id, ie.revision), (), [])

226

invdelta.append((base_inv.id2path(ie.file_id), path, ie.file_id, ie))

227

# Remember for next time

228

existing_children = set()

229

child_modes = {}

230

shamap = []

231

tree = lookup_object(hexsha)

232

for mode, name, child_hexsha in tree.entries():

233

basename = name.decode("utf-8")

234

existing_children.add(basename)

235

child_path = osutils.pathjoin(path, name)

236

if stat.S_ISDIR(mode):

237

subinvdelta, grandchildmodes, subshamap = import_git_tree(

238

texts, mapping, child_path, child_hexsha, base_inv,

239

file_id, revision_id, parent_invs, shagitmap, lookup_object)

240

invdelta.extend(subinvdelta)

241

child_modes.update(grandchildmodes)

242

shamap.extend(subshamap)

243

elif S_ISGITLINK(mode): # submodule

244

subinvdelta, grandchildmodes, subshamap = import_git_submodule(

245

texts, mapping, child_path, child_hexsha, base_inv,

246

file_id, revision_id, parent_invs, shagitmap, lookup_object)

247

invdelta.extend(subinvdelta)

248

child_modes.update(grandchildmodes)

249

shamap.extend(subshamap)

250

else:

251

subinvdelta, subshamap = import_git_blob(texts, mapping,

252

child_path, child_hexsha, base_inv, file_id, revision_id,

253

parent_invs, shagitmap, lookup_object,

254

mode_is_executable(mode), stat.S_ISLNK(mode))

255

invdelta.extend(subinvdelta)

256

shamap.extend(subshamap)

257

if mode not in (stat.S_IFDIR, DEFAULT_FILE_MODE,

258

stat.S_IFLNK, DEFAULT_FILE_MODE|0111):

259

child_modes[child_path] = mode

260

# Remove any children that have disappeared

261

invdelta.extend(remove_disappeared_children(base_inv, base_ie, existing_children))

262

shamap.append((hexsha, "tree", (file_id, revision_id)))

263

return invdelta, child_modes, shamap

264

265

266

def import_git_objects(repo, mapping, object_iter, target_git_object_retriever,

267

heads, pb=None):

268

"""Import a set of git objects into a bzr repository.

269

270

:param repo: Target Bazaar repository

271

:param mapping: Mapping to use

272

:param object_iter: Iterator over Git objects.

273

"""

274

def lookup_object(sha):

275

try:

276

return object_iter[sha]

277

except KeyError:

278

return target_git_object_retriever[sha]

279

# TODO: a more (memory-)efficient implementation of this

280

graph = []

281

root_trees = {}

282

revisions = {}

283

checked = set()

284

heads = list(heads)

285

parent_invs_cache = LRUCache(50)

286

# Find and convert commit objects

287

while heads:

288

if pb is not None:

289

pb.update("finding revisions to fetch", len(graph), None)

290

head = heads.pop()

291

assert isinstance(head, str)

292

try:

293

o = lookup_object(head)

294

except KeyError:

295

continue

296

if isinstance(o, Commit):

297

rev = mapping.import_commit(o)

298

if repo.has_revision(rev.revision_id):

299

continue

300

squash_revision(repo, rev)

301

root_trees[rev.revision_id] = o.tree

302

revisions[rev.revision_id] = rev

303

graph.append((rev.revision_id, rev.parent_ids))

304

target_git_object_retriever._idmap.add_entry(o.id, "commit",

305

(rev.revision_id, o.tree))

306

heads.extend([p for p in o.parents if p not in checked])

307

elif isinstance(o, Tag):

308

heads.append(o.object[1])

309

else:

310

trace.warning("Unable to import head object %r" % o)

311

checked.add(head)

312

# Order the revisions

313

# Create the inventory objects

314

for i, revid in enumerate(topo_sort(graph)):

315

if pb is not None:

316

pb.update("fetching revisions", i, len(graph))

317

rev = revisions[revid]

318

# We have to do this here, since we have to walk the tree and

319

# we need to make sure to import the blobs / trees with the right

320

# path; this may involve adding them more than once.

321

parent_invs = []

322

for parent_id in rev.parent_ids:

323

try:

324

parent_invs.append(parent_invs_cache[parent_id])

325

except KeyError:

326

parent_inv = repo.get_inventory(parent_id)

327

parent_invs.append(parent_inv)

328

parent_invs_cache[parent_id] = parent_inv

329

if parent_invs == []:

330

base_inv = Inventory(root_id=None)

331

else:

332

base_inv = parent_invs[0]

333

inv_delta, unusual_modes, shamap = import_git_tree(repo.texts,

334

mapping, "", root_trees[revid], base_inv, None, revid,

335

parent_invs, target_git_object_retriever._idmap, lookup_object)

336

target_git_object_retriever._idmap.add_entries(shamap)

337

if unusual_modes != {}:

338

for path, mode in unusual_modes.iteritems():

339

warn_unusual_mode(rev.foreign_revid, path, mode)

340

mapping.import_unusual_file_modes(rev, unusual_modes)

341

try:

342

basis_id = rev.parent_ids[0]

343

except IndexError:

344

basis_id = NULL_REVISION

345

rev.inventory_sha1, inv = repo.add_inventory_by_delta(basis_id,

346

inv_delta, rev.revision_id, rev.parent_ids)

347

parent_invs_cache[rev.revision_id] = inv

348

repo.add_revision(rev.revision_id, rev)

349

if "verify" in debug.debug_flags:

350

new_unusual_modes = mapping.export_unusual_file_modes(rev)

351

if new_unusual_modes != unusual_modes:

352

raise AssertionError("unusual modes don't match: %r != %r" % (unusual_modes, new_unusual_modes))

353

objs = inventory_to_tree_and_blobs(inv, repo.texts, mapping, unusual_modes)

354

for sha1, newobj, path in objs:

355

assert path is not None

356

oldobj = tree_lookup_path(lookup_object, root_trees[revid], path)

357

if oldobj != newobj:

358

raise AssertionError("%r != %r in %s" % (oldobj, newobj, path))

359

360

target_git_object_retriever._idmap.commit()

361

362

363

class InterGitRepository(InterRepository):

364

365

_matching_repo_format = GitRepositoryFormat()

366

367

@staticmethod

368

def _get_repo_format_to_test():

369

return None

370

371

def copy_content(self, revision_id=None, pb=None):

372

"""See InterRepository.copy_content."""

373

self.fetch(revision_id, pb, find_ghosts=False)

374

375

def fetch(self, revision_id=None, pb=None, find_ghosts=False, mapping=None,

376

fetch_spec=None):

377

self.fetch_refs(revision_id=revision_id, pb=pb, find_ghosts=find_ghosts,

378

mapping=mapping, fetch_spec=fetch_spec)

379

380

381

class InterGitNonGitRepository(InterGitRepository):

382

"""Base InterRepository that copies revisions from a Git into a non-Git

383

repository."""

384

385

def fetch_refs(self, revision_id=None, pb=None, find_ghosts=False,

386

mapping=None, fetch_spec=None):

387

if mapping is None:

388

mapping = self.source.get_mapping()

389

if revision_id is not None:

390

interesting_heads = [revision_id]

391

elif fetch_spec is not None:

392

interesting_heads = fetch_spec.heads

393

else:

394

interesting_heads = None

395

self._refs = {}

396

def determine_wants(refs):

397

self._refs = refs

398

if interesting_heads is None:

399

ret = [sha for (ref, sha) in refs.iteritems() if not ref.endswith("^{}")]

400

else:

401

ret = [mapping.revision_id_bzr_to_foreign(revid)[0] for revid in interesting_heads if revid not in (None, NULL_REVISION)]

402

return [rev for rev in ret if not self.target.has_revision(mapping.revision_id_foreign_to_bzr(rev))]

403

self.fetch_objects(determine_wants, mapping, pb)

404

return self._refs

405

406

407

class InterRemoteGitNonGitRepository(InterGitNonGitRepository):

408

"""InterRepository that copies revisions from a remote Git into a non-Git

409

repository."""

410

411

def fetch_objects(self, determine_wants, mapping, pb=None):

412

def progress(text):

413

pb.update("git: %s" % text.rstrip("\r\n"), 0, 0)

414

store = BazaarObjectStore(self.target, mapping)

415

self.target.lock_write()

416

try:

417

heads = self.target.get_graph().heads(self.target.all_revision_ids())

418

graph_walker = store.get_graph_walker(

419

[store._lookup_revision_sha1(head) for head in heads])

420

recorded_wants = []

421

422

def record_determine_wants(heads):

423

wants = determine_wants(heads)

424

recorded_wants.extend(wants)

425

return wants

426

427

create_pb = None

428

if pb is None:

429

create_pb = pb = ui.ui_factory.nested_progress_bar()

430

try:

431

self.target.start_write_group()

432

try:

433

objects_iter = self.source.fetch_objects(

434

record_determine_wants, graph_walker,

435

store.get_raw, progress)

436

import_git_objects(self.target, mapping, objects_iter,

437

store, recorded_wants, pb)

438

finally:

439

self.target.commit_write_group()

440

finally:

441

if create_pb:

442

create_pb.finished()

443

finally:

444

self.target.unlock()

445

446

@staticmethod

447

def is_compatible(source, target):

448

"""Be compatible with GitRepository."""

449

# FIXME: Also check target uses VersionedFile

450

return (isinstance(source, RemoteGitRepository) and

451

target.supports_rich_root() and

452

not isinstance(target, GitRepository))

453

454

455

class InterLocalGitNonGitRepository(InterGitNonGitRepository):

456

"""InterRepository that copies revisions from a local Git into a non-Git

457

repository."""

458

459

def fetch_objects(self, determine_wants, mapping, pb=None):

460

wants = determine_wants(self.source._git.get_refs())

461

create_pb = None

462

if pb is None:

463

create_pb = pb = ui.ui_factory.nested_progress_bar()

464

target_git_object_retriever = BazaarObjectStore(self.target, mapping)

465

try:

466

self.target.lock_write()

467

try:

468

self.target.start_write_group()

469

try:

470

import_git_objects(self.target, mapping,

471

self.source._git.object_store,

472

target_git_object_retriever, wants, pb)

473

finally:

474

self.target.commit_write_group()

475

finally:

476

self.target.unlock()

477

finally:

478

if create_pb:

479

create_pb.finished()

480

481

@staticmethod

482

def is_compatible(source, target):

483

"""Be compatible with GitRepository."""

484

# FIXME: Also check target uses VersionedFile

485

return (isinstance(source, LocalGitRepository) and

486

target.supports_rich_root() and

487

not isinstance(target, GitRepository))

488

489

490

class InterGitGitRepository(InterGitRepository):

491

"""InterRepository that copies between Git repositories."""

492

493

def fetch_refs(self, revision_id=None, pb=None, find_ghosts=False,

494

mapping=None, fetch_spec=None, branches=None):

495

if mapping is None:

496

mapping = self.source.get_mapping()

497

def progress(text):

498

trace.info("git: %s", text)

499

r = self.target._git

500

if revision_id is not None:

501

args = [mapping.revision_id_bzr_to_foreign(revision_id)[0]]

502

elif fetch_spec is not None:

503

args = [mapping.revision_id_bzr_to_foreign(revid)[0] for revid in fetch_spec.heads]

504

if branches is not None:

505

determine_wants = lambda x: [x[y] for y in branches if not x[y] in r.object_store]

506

elif fetch_spec is None and revision_id is None:

507

determine_wants = r.object_store.determine_wants_all

508

else:

509

determine_wants = lambda x: [y for y in args if not y in r.object_store]

510

511

graphwalker = r.get_graph_walker()

512

f, commit = r.object_store.add_thin_pack()

513

try:

514

refs = self.source.fetch_pack(determine_wants, graphwalker,

515

f.write, progress)

516

commit()

517

return refs

518

except:

519

f.close()

520

raise

521

522

@staticmethod

523

def is_compatible(source, target):

524

"""Be compatible with GitRepository."""

525

return (isinstance(source, GitRepository) and

526

isinstance(target, GitRepository))