# Copyright (C) 2002-2006 Stephen Kennedy <stevek@gnome.org>

# Copyright (C) 2009-2019 Kai Willadsen <kai.willadsen@gmail.com>

#

# This program is free software: you can redistribute it and/or modify

# it under the terms of the GNU General Public License as published by

# the Free Software Foundation, either version 2 of the License, or (at

# your option) any later version.

#

# This program is distributed in the hope that it will be useful, but

# WITHOUT ANY WARRANTY; without even the implied warranty of

# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU

# General Public License for more details.

#

# You should have received a copy of the GNU General Public License

# along with this program. If not, see <http://www.gnu.org/licenses/>.

import collections

import copy

import errno

import functools

import os

import shutil

import stat

import sys

from collections import namedtuple

from decimal import Decimal

from mmap import ACCESS_COPY, mmap

from gi.repository import Gdk, Gio, GLib, GObject, Gtk

# TODO: Don't from-import whole modules

from meld import misc, tree

from meld.conf import _

from meld.const import FILE_FILTER_ACTION_FORMAT, MISSING_TIMESTAMP

from meld.iohelpers import trash_or_confirm

from meld.melddoc import MeldDoc

from meld.misc import all_same, apply_text_filters, with_focused_pane

from meld.recent import RecentType

from meld.settings import bind_settings, get_meld_settings, settings

from meld.treehelpers import refocus_deleted_path, tree_path_as_tuple

from meld.ui.cellrenderers import (

CellRendererByteSize,

CellRendererDate,

CellRendererFileMode,

)

from meld.ui.emblemcellrenderer import EmblemCellRenderer

from meld.ui.util import map_widgets_into_lists

class StatItem(namedtuple('StatItem', 'mode size time')):

__slots__ = ()

@classmethod

def _make(cls, stat_result):

return StatItem(stat.S_IFMT(stat_result.st_mode),

stat_result.st_size, stat_result.st_mtime)

def shallow_equal(self, other, time_resolution_ns):

if self.size != other.size:

return False

# Shortcut to avoid expensive Decimal calculations. 2 seconds is our

# current accuracy threshold (for VFAT), so should be safe for now.

if abs(self.time - other.time) > 2:

return False

dectime1 = Decimal(self.time).scaleb(Decimal(9)).quantize(1)

dectime2 = Decimal(other.time).scaleb(Decimal(9)).quantize(1)

mtime1 = dectime1 // time_resolution_ns

mtime2 = dectime2 // time_resolution_ns

return mtime1 == mtime2

CacheResult = namedtuple('CacheResult', 'stats result')

_cache = {}

Same, SameFiltered, DodgySame, DodgyDifferent, Different, FileError = \

list(range(6))

# TODO: Get the block size from os.stat

CHUNK_SIZE = 4096

def remove_blank_lines(text):

"""

Remove blank lines from text.

And normalize line ending

"""

return b'\n'.join(filter(bool, text.splitlines()))

def _files_contents(files, stats):

mmaps = []

is_bin = False

contents = [b'' for file_obj in files]

for index, file_and_stat in enumerate(zip(files, stats)):

file_obj, stat_ = file_and_stat

# use mmap for files with size > CHUNK_SIZE

data = b''

if stat_.size > CHUNK_SIZE:

data = mmap(file_obj.fileno(), 0, access=ACCESS_COPY)

mmaps.append(data)

else:

data = file_obj.read()

contents[index] = data

# Rough test to see whether files are binary.

chunk_size = min([stat_.size, CHUNK_SIZE])

if b"\0" in data[:chunk_size]:

is_bin = True

return contents, mmaps, is_bin

def _contents_same(contents, file_size):

other_files_index = list(range(1, len(contents)))

chunk_range = zip(

range(0, file_size, CHUNK_SIZE),

range(CHUNK_SIZE, file_size + CHUNK_SIZE, CHUNK_SIZE)

)

for start, end in chunk_range:

chunk = contents[0][start:end]

for index in other_files_index:

if not chunk == contents[index][start:end]:

return Different

def _normalize(contents, ignore_blank_lines, regexes=()):

contents = (bytes(c) for c in contents)

# For probable text files, discard newline differences to match

if ignore_blank_lines:

contents = (remove_blank_lines(c) for c in contents)

else:

contents = (b"\n".join(c.splitlines()) for c in contents)

if regexes:

contents = (apply_text_filters(c, regexes) for c in contents)

if ignore_blank_lines:

# We re-remove blank lines here in case applying text

# filters has caused more lines to be blank.

contents = (remove_blank_lines(c) for c in contents)

return contents

def _files_same(files, regexes, comparison_args):

"""Determine whether a list of files are the same.

Possible results are:

Same: The files are the same

SameFiltered: The files are identical only after filtering with 'regexes'

DodgySame: The files are superficially the same (i.e., type, size, mtime)

DodgyDifferent: The files are superficially different

FileError: There was a problem reading one or more of the files

"""

if all_same(files):

return Same

files = tuple(files)

stats = tuple([StatItem._make(os.stat(f)) for f in files])

shallow_comparison = comparison_args['shallow-comparison']

time_resolution_ns = comparison_args['time-resolution']

ignore_blank_lines = comparison_args['ignore_blank_lines']

apply_text_filters = comparison_args['apply-text-filters']

need_contents = ignore_blank_lines or apply_text_filters

regexes = tuple(regexes) if apply_text_filters else ()

# If all entries are directories, they are considered to be the same

if all([stat.S_ISDIR(s.mode) for s in stats]):

return Same

# If any entries are not regular files, consider them different

if not all([stat.S_ISREG(s.mode) for s in stats]):

return Different

# Compare files superficially if the options tells us to

if shallow_comparison:

all_same_timestamp = all(

s.shallow_equal(stats[0], time_resolution_ns) for s in stats[1:]

)

return DodgySame if all_same_timestamp else Different

same_size = all_same([s.size for s in stats])

# If there are no text filters, unequal sizes imply a difference

if not need_contents and not same_size:

return Different

# Check the cache before doing the expensive comparison

cache_key = (files, need_contents, regexes, ignore_blank_lines)

cache = _cache.get(cache_key)

if cache and cache.stats == stats:

return cache.result

# Open files and compare bit-by-bit

result = None

try:

mmaps = []

handles = [open(file_path, "rb") for file_path in files]

try:

contents, mmaps, is_bin = _files_contents(handles, stats)

# compare files chunk-by-chunk

if same_size:

result = _contents_same(contents, stats[0].size)

else:

result = Different

# normalize and compare files again

if result == Different and need_contents and not is_bin:

contents = _normalize(contents, ignore_blank_lines, regexes)

result = SameFiltered if all_same(contents) else Different

# Files are too large; we can't apply filters

except (MemoryError, OverflowError):

result = DodgySame if all_same(stats) else DodgyDifferent

finally:

for m in mmaps:

m.close()

for h in handles:

h.close()

except IOError:

# Don't cache generic errors as results

return FileError

if result is None:

result = Same

_cache[cache_key] = CacheResult(stats, result)

return result

EMBLEM_NEW = "emblem-new"

EMBLEM_SYMLINK = "emblem-symbolic-link"

COL_EMBLEM, COL_EMBLEM_SECONDARY, COL_SIZE, COL_TIME, COL_PERMS, COL_END = \

range(tree.COL_END, tree.COL_END + 6)

class DirDiffTreeStore(tree.DiffTreeStore):

def __init__(self, ntree):

# FIXME: size should be a GObject.TYPE_UINT64, but we use -1 as a flag

super().__init__(ntree, [str, str, GObject.TYPE_INT64, float, int])

def add_error(self, parent, msg, pane):

defaults = {

COL_TIME: MISSING_TIMESTAMP,

COL_SIZE: -1,

COL_PERMS: -1

}

super().add_error(parent, msg, pane, defaults)

class CanonicalListing:

"""Multi-pane lists with canonicalised matching and error detection"""

def __init__(self, n, canonicalize=None):

self.items = collections.defaultdict(lambda: [None] * n)

self.errors = []

self.canonicalize = canonicalize

self.add = self.add_simple if canonicalize is None else self.add_canon

def add_simple(self, pane, item):

self.items[item][pane] = item

def add_canon(self, pane, item):

ci = self.canonicalize(item)

if self.items[ci][pane] is None:

self.items[ci][pane] = item

else:

self.errors.append((pane, item, self.items[ci][pane]))

def get(self):

def filled(seq):

fill_value = next(s for s in seq if s)

return tuple(s or fill_value for s in seq)

return sorted(filled(v) for v in self.items.values())

@staticmethod

def canonicalize_lower(element):

return element.lower()

@Gtk.Template(resource_path='/org/gnome/meld/ui/dirdiff.ui')

class DirDiff(Gtk.VBox, tree.TreeviewCommon, MeldDoc):

__gtype_name__ = "DirDiff"

close_signal = MeldDoc.close_signal

create_diff_signal = MeldDoc.create_diff_signal

file_changed_signal = MeldDoc.file_changed_signal

label_changed = MeldDoc.label_changed

move_diff = MeldDoc.move_diff

tab_state_changed = MeldDoc.tab_state_changed

__gsettings_bindings__ = (

('folder-ignore-symlinks', 'ignore-symlinks'),

('folder-shallow-comparison', 'shallow-comparison'),

('folder-time-resolution', 'time-resolution'),

('folder-status-filters', 'status-filters'),

('folder-filter-text', 'apply-text-filters'),

('ignore-blank-lines', 'ignore-blank-lines'),

)

apply_text_filters = GObject.Property(

type=bool,

nick="Apply text filters",

blurb=(

"Whether text filters and other text sanitisation preferences "

"should be applied when comparing file contents"),

default=False,

)

ignore_blank_lines = GObject.Property(

type=bool,

nick="Ignore blank lines",

blurb="Whether to ignore blank lines when comparing file contents",

default=False,

)

ignore_symlinks = GObject.Property(

type=bool,

nick="Ignore symbolic links",

blurb="Whether to follow symbolic links when comparing folders",

default=False,

)

shallow_comparison = GObject.Property(

type=bool,

nick="Use shallow comparison",

blurb="Whether to compare files based solely on size and mtime",

default=False,

)

status_filters = GObject.Property(

type=GObject.TYPE_STRV,

nick="File status filters",

blurb="Files with these statuses will be shown by the comparison.",

)

time_resolution = GObject.Property(

type=int,

nick="Time resolution",

blurb="When comparing based on mtime, the minimum difference in "

"nanoseconds between two files before they're considered to "

"have different mtimes.",

default=100,

)

show_overview_map = GObject.Property(type=bool, default=True)

chunkmap0 = Gtk.Template.Child()

chunkmap1 = Gtk.Template.Child()

chunkmap2 = Gtk.Template.Child()

treeview0 = Gtk.Template.Child()

treeview1 = Gtk.Template.Child()

treeview2 = Gtk.Template.Child()

fileentry0 = Gtk.Template.Child()

fileentry1 = Gtk.Template.Child()

fileentry2 = Gtk.Template.Child()

scrolledwindow0 = Gtk.Template.Child()

scrolledwindow1 = Gtk.Template.Child()

scrolledwindow2 = Gtk.Template.Child()

linkmap0 = Gtk.Template.Child()

linkmap1 = Gtk.Template.Child()

msgarea_mgr0 = Gtk.Template.Child()

msgarea_mgr1 = Gtk.Template.Child()

msgarea_mgr2 = Gtk.Template.Child()

overview_map_revealer = Gtk.Template.Child()

vbox0 = Gtk.Template.Child()

vbox1 = Gtk.Template.Child()

vbox2 = Gtk.Template.Child()

dummy_toolbar_overview_map = Gtk.Template.Child()

dummy_toolbar_linkmap0 = Gtk.Template.Child()

dummy_toolbar_linkmap1 = Gtk.Template.Child()

file_toolbar0 = Gtk.Template.Child()

file_toolbar1 = Gtk.Template.Child()

file_toolbar2 = Gtk.Template.Child()

state_actions = {

tree.STATE_NORMAL: ("normal", "folder-status-same"),

tree.STATE_NOCHANGE: ("normal", "folder-status-same"),

tree.STATE_NEW: ("new", "folder-status-new"),

tree.STATE_MODIFIED: ("modified", "folder-status-modified"),

}

def __init__(self, num_panes):

super().__init__()

# FIXME:

# This unimaginable hack exists because GObject (or GTK+?)

# doesn't actually correctly chain init calls, even if they're

# not to GObjects. As a workaround, we *should* just be able to

# put our class first, but because of Gtk.Template we can't do

# that if it's a GObject, because GObject doesn't support

# multiple inheritance and we need to inherit from our Widget

# parent to make Template work.

MeldDoc.__init__(self)

bind_settings(self)

self.view_action_group = Gio.SimpleActionGroup()

property_actions = (

('show-overview-map', self, 'show-overview-map'),

)

for action_name, obj, prop_name in property_actions:

action = Gio.PropertyAction.new(action_name, obj, prop_name)

self.view_action_group.add_action(action)

# Manually handle GAction additions

actions = (

('find', self.action_find),

('folder-collapse', self.action_folder_collapse),

('folder-compare', self.action_diff),

('folder-copy-left', self.action_copy_left),

('folder-copy-right', self.action_copy_right),

('folder-delete', self.action_delete),

('folder-expand', self.action_folder_expand),

('next-change', self.action_next_change),

('next-pane', self.action_next_pane),

('open-external', self.action_open_external),

('previous-change', self.action_previous_change),

('previous-pane', self.action_prev_pane),

('refresh', self.action_refresh),

('copy-file-paths', self.action_copy_file_paths),

)

for name, callback in actions:

action = Gio.SimpleAction.new(name, None)

action.connect('activate', callback)

self.view_action_group.add_action(action)

actions = (

("folder-status-same", self.action_filter_state_change,

GLib.Variant.new_boolean(False)),

("folder-status-new", self.action_filter_state_change,

GLib.Variant.new_boolean(False)),

("folder-status-modified", self.action_filter_state_change,

GLib.Variant.new_boolean(False)),

("folder-ignore-case", self.action_ignore_case_change,

GLib.Variant.new_boolean(False)),

)

for (name, callback, state) in actions:

action = Gio.SimpleAction.new_stateful(name, None, state)

action.connect('change-state', callback)

self.view_action_group.add_action(action)

builder = Gtk.Builder.new_from_resource(

'/org/gnome/meld/ui/dirdiff-menus.ui')

context_menu = builder.get_object('dirdiff-context-menu')

self.popup_menu = Gtk.Menu.new_from_model(context_menu)

self.popup_menu.attach_to_widget(self)

builder = Gtk.Builder.new_from_resource(

'/org/gnome/meld/ui/dirdiff-actions.ui')

self.toolbar_actions = builder.get_object('view-toolbar')

self.name_filters = []

self.text_filters = []

self.create_name_filters()

self.create_text_filters()

meld_settings = get_meld_settings()

self.settings_handlers = [

meld_settings.connect(

"file-filters-changed", self.on_file_filters_changed),

meld_settings.connect(

"text-filters-changed", self.on_text_filters_changed)

]

# Handle overview map visibility binding

self.bind_property(

'show-overview-map', self.overview_map_revealer, 'reveal-child',

GObject.BindingFlags.DEFAULT | GObject.BindingFlags.SYNC_CREATE,

)

self.overview_map_revealer.bind_property(

'child-revealed', self.dummy_toolbar_overview_map, 'visible')

map_widgets_into_lists(

self,

[

"treeview", "fileentry", "scrolledwindow", "chunkmap",

"linkmap", "msgarea_mgr", "vbox", "dummy_toolbar_linkmap",

"file_toolbar",

]

)

self.ensure_style()

self.custom_labels = []

self.set_num_panes(num_panes)

self.connect("style-updated", self.model.on_style_updated)

self.model.on_style_updated(self)

self.do_to_others_lock = False

for treeview in self.treeview:

treeview.set_search_equal_func(tree.treeview_search_cb, None)

self.force_cursor_recalculate = False

self.current_path, self.prev_path, self.next_path = None, None, None

self.focus_pane = None

self.row_expansions = set()

# One column-dict for each treeview, for changing visibility and order

self.columns_dict = [{}, {}, {}]

for i in range(3):

col_index = self.model.column_index

# Create icon and filename CellRenderer

column = Gtk.TreeViewColumn(_("Name"))

column.set_resizable(True)

rentext = Gtk.CellRendererText()

renicon = EmblemCellRenderer()

column.pack_start(renicon, False)

column.pack_start(rentext, True)

column.set_attributes(rentext, markup=col_index(tree.COL_TEXT, i),

foreground_rgba=col_index(tree.COL_FG, i),

style=col_index(tree.COL_STYLE, i),

weight=col_index(tree.COL_WEIGHT, i),

strikethrough=col_index(tree.COL_STRIKE, i))

column.set_attributes(

renicon,

icon_name=col_index(tree.COL_ICON, i),

emblem_name=col_index(COL_EMBLEM, i),

secondary_emblem_name=col_index(COL_EMBLEM_SECONDARY, i),

icon_tint=col_index(tree.COL_TINT, i)

)

self.treeview[i].append_column(column)

self.columns_dict[i]["name"] = column

# Create file size CellRenderer

column = Gtk.TreeViewColumn(_("Size"))

column.set_resizable(True)

rentext = CellRendererByteSize()

column.pack_start(rentext, True)

column.set_attributes(rentext, bytesize=col_index(COL_SIZE, i))

self.treeview[i].append_column(column)

self.columns_dict[i]["size"] = column

# Create date-time CellRenderer

column = Gtk.TreeViewColumn(_("Modification time"))

column.set_resizable(True)

rentext = CellRendererDate()

column.pack_start(rentext, True)

column.set_attributes(rentext, timestamp=col_index(COL_TIME, i))

self.treeview[i].append_column(column)

self.columns_dict[i]["modification time"] = column

# Create permissions CellRenderer

column = Gtk.TreeViewColumn(_("Permissions"))

column.set_resizable(True)

rentext = CellRendererFileMode()

column.pack_start(rentext, False)

column.set_attributes(rentext, file_mode=col_index(COL_PERMS, i))

self.treeview[i].append_column(column)

self.columns_dict[i]["permissions"] = column

for i in range(3):

selection = self.treeview[i].get_selection()

selection.set_mode(Gtk.SelectionMode.MULTIPLE)

selection.connect('changed', self.on_treeview_selection_changed, i)

self.scrolledwindow[i].get_vadjustment().connect(

"value-changed", self._sync_vscroll)

self.scrolledwindow[i].get_hadjustment().connect(

"value-changed", self._sync_hscroll)

self.linediffs = [[], []]

self.update_treeview_columns(settings, 'folder-columns')

settings.connect('changed::folder-columns',

self.update_treeview_columns)

self.update_comparator()

self.connect("notify::shallow-comparison", self.update_comparator)

self.connect("notify::time-resolution", self.update_comparator)

self.connect("notify::ignore-blank-lines", self.update_comparator)

self.connect("notify::apply-text-filters", self.update_comparator)

# The list copying and state_filters reset here is because the action

# toggled callback modifies the state while we're constructing it.

self.state_filters = []

state_filters = []

for s in self.state_actions:

if self.state_actions[s][0] in self.props.status_filters:

state_filters.append(s)

action_name = self.state_actions[s][1]

self.set_action_state(

action_name, GLib.Variant.new_boolean(True))

self.state_filters = state_filters

self._scan_in_progress = 0

def queue_draw(self):

for treeview in self.treeview:

treeview.queue_draw()

def update_comparator(self, *args):

comparison_args = {

'shallow-comparison': self.props.shallow_comparison,

'time-resolution': self.props.time_resolution,

'apply-text-filters': self.props.apply_text_filters,

'ignore_blank_lines': self.props.ignore_blank_lines,

}

self.file_compare = functools.partial(

_files_same, comparison_args=comparison_args)

self.refresh()

def update_treeview_columns(self, settings, key):

"""Update the visibility and order of columns"""

columns = settings.get_value(key)

for i, treeview in enumerate(self.treeview):

extra_cols = False

last_column = treeview.get_column(0)

for column_name, visible in columns:

extra_cols = extra_cols or visible

current_column = self.columns_dict[i][column_name]

current_column.set_visible(visible)

treeview.move_column_after(current_column, last_column)

last_column = current_column

treeview.set_headers_visible(extra_cols)

def on_file_filters_changed(self, app):

relevant_change = self.create_name_filters()

if relevant_change:

self.refresh()

def create_name_filters(self):

meld_settings = get_meld_settings()

# Ordering of name filters is irrelevant

old_active = set([f.filter_string for f in self.name_filters

if f.active])

new_active = set([f.filter_string for f in meld_settings.file_filters

if f.active])

active_filters_changed = old_active != new_active

# TODO: Rework name_filters to use a map-like structure so that we

# don't need _action_name_filter_map.

self._action_name_filter_map = {}

self.name_filters = [copy.copy(f) for f in meld_settings.file_filters]

for i, filt in enumerate(self.name_filters):

action = Gio.SimpleAction.new_stateful(

name=FILE_FILTER_ACTION_FORMAT.format(i),

parameter_type=None,

state=GLib.Variant.new_boolean(filt.active),

)

action.connect('change-state', self._update_name_filter)

action.set_enabled(filt.filter is not None)

self.view_action_group.add_action(action)

self._action_name_filter_map[action] = filt

return active_filters_changed

def on_text_filters_changed(self, app):

relevant_change = self.create_text_filters()

if relevant_change:

self.refresh()

def create_text_filters(self):

meld_settings = get_meld_settings()

# In contrast to file filters, ordering of text filters can matter

old_active = [f.filter_string for f in self.text_filters if f.active]

new_active = [f.filter_string for f in meld_settings.text_filters

if f.active]

active_filters_changed = old_active != new_active

self.text_filters = [copy.copy(f) for f in meld_settings.text_filters]

return active_filters_changed

def _do_to_others(self, master, objects, methodname, args):

if self.do_to_others_lock:

return

self.do_to_others_lock = True

try:

others = [o for o in objects[:self.num_panes] if o != master]

for o in others:

method = getattr(o, methodname)

method(*args)

finally:

self.do_to_others_lock = False

def _sync_vscroll(self, adjustment):

adjs = [sw.get_vadjustment() for sw in self.scrolledwindow]

self._do_to_others(

adjustment, adjs, "set_value", (int(adjustment.get_value()),))

def _sync_hscroll(self, adjustment):

adjs = [sw.get_hadjustment() for sw in self.scrolledwindow]

self._do_to_others(

adjustment, adjs, "set_value", (int(adjustment.get_value()),))

def _get_focused_pane(self):

for i, treeview in enumerate(self.treeview):

if treeview.is_focus():

return i

return None

def file_deleted(self, path, pane):

# is file still extant in other pane?

it = self.model.get_iter(path)

files = self.model.value_paths(it)

is_present = [os.path.exists(f) for f in files]

if 1 in is_present:

self._update_item_state(it)

else: # nope its gone

self.model.remove(it)

def file_created(self, path, pane):

it = self.model.get_iter(path)

root = Gtk.TreePath.new_first()

while it and self.model.get_path(it) != root:

self._update_item_state(it)

it = self.model.iter_parent(it)

@Gtk.Template.Callback()

def on_fileentry_file_set(self, entry):

files = [e.get_file() for e in self.fileentry[:self.num_panes]]

paths = [f.get_path() for f in files]

self.set_locations(paths)

def set_locations(self, locations):

self.set_num_panes(len(locations))

# This is difficult to trigger, and to test. Most of the time here we

# will actually have had UTF-8 from GTK, which has been unicode-ed by

# the time we get this far. This is a fallback, and may be wrong!

locations = list(locations)

for i, l in enumerate(locations):

if l and not isinstance(l, str):

locations[i] = l.decode(sys.getfilesystemencoding())

locations = [os.path.abspath(l) if l else '' for l in locations]

self.current_path = None

self.model.clear()

for m in self.msgarea_mgr:

m.clear()

for pane, loc in enumerate(locations):

if loc:

self.fileentry[pane].set_filename(loc)

child = self.model.add_entries(None, locations)

self.treeview0.grab_focus()

self._update_item_state(child)

self.recompute_label()

self.scheduler.remove_all_tasks()

self.recursively_update(Gtk.TreePath.new_first())

def get_comparison(self):

root = self.model.get_iter_first()

if root:

uris = [Gio.File.new_for_path(d)

for d in self.model.value_paths(root)]

else:

uris = []

return RecentType.Folder, uris

def recursively_update(self, path):

"""Recursively update from tree path 'path'.

"""

it = self.model.get_iter(path)

child = self.model.iter_children(it)

while child:

self.model.remove(child)

child = self.model.iter_children(it)

self._update_item_state(it)

self._scan_in_progress += 1

self.scheduler.add_task(self._search_recursively_iter(path))

def _search_recursively_iter(self, rootpath):

for t in self.treeview:

sel = t.get_selection()

sel.unselect_all()

yield _("[%s] Scanning %s") % (self.label_text, "")

prefixlen = 1 + len(

self.model.value_path(self.model.get_iter(rootpath), 0))

symlinks_followed = set()

# TODO: This is horrible.

if isinstance(rootpath, tuple):

rootpath = Gtk.TreePath(rootpath)

todo = [rootpath]

expanded = set()

shadowed_entries = []

invalid_filenames = []

while len(todo):

todo.sort() # depth first

path = todo.pop(0)

it = self.model.get_iter(path)

roots = self.model.value_paths(it)

# Buggy ordering when deleting rows means that we sometimes try to

# recursively update files; this fix seems the least invasive.

if not any(os.path.isdir(root) for root in roots):

continue

yield _("[%s] Scanning %s") % (

self.label_text, roots[0][prefixlen:])

differences = False

encoding_errors = []

canonicalize = None

# TODO: Map this to a GObject prop instead?

if self.get_action_state('folder-ignore-case'):

canonicalize = CanonicalListing.canonicalize_lower

dirs = CanonicalListing(self.num_panes, canonicalize)

files = CanonicalListing(self.num_panes, canonicalize)

for pane, root in enumerate(roots):

if not os.path.isdir(root):

continue

try:

entries = os.listdir(root)

except OSError as err:

self.model.add_error(it, err.strerror, pane)

differences = True

continue

for f in self.name_filters:

if not f.active or f.filter is None:

continue

entries = [e for e in entries if f.filter.match(e) is None]

for e in entries:

try:

e.encode('utf8')

except UnicodeEncodeError:

invalid = e.encode('utf8', 'surrogatepass')

printable = invalid.decode('utf8', 'backslashreplace')

encoding_errors.append((pane, printable))

continue

try:

s = os.lstat(os.path.join(root, e))

# Covers certain unreadable symlink cases; see bgo#585895

except OSError as err:

error_string = e + err.strerror

self.model.add_error(it, error_string, pane)

continue

if stat.S_ISLNK(s.st_mode):

if self.props.ignore_symlinks:

continue

key = (s.st_dev, s.st_ino)

if key in symlinks_followed:

continue

symlinks_followed.add(key)

try:

s = os.stat(os.path.join(root, e))

if stat.S_ISREG(s.st_mode):

files.add(pane, e)

elif stat.S_ISDIR(s.st_mode):

dirs.add(pane, e)

except OSError as err:

if err.errno == errno.ENOENT:

error_string = e + ": Dangling symlink"

else:

error_string = e + err.strerror

self.model.add_error(it, error_string, pane)

differences = True

elif stat.S_ISREG(s.st_mode):

files.add(pane, e)

elif stat.S_ISDIR(s.st_mode):

dirs.add(pane, e)

else:

# FIXME: Unhandled stat type

pass

for pane, f in encoding_errors:

invalid_filenames.append((pane, roots[pane], f))

for pane, f1, f2 in dirs.errors + files.errors:

shadowed_entries.append((pane, roots[pane], f1, f2))

alldirs = self._filter_on_state(roots, dirs.get())

allfiles = self._filter_on_state(roots, files.get())

if alldirs or allfiles:

for names in alldirs:

entries = [

os.path.join(r, n) for r, n in zip(roots, names)]

child = self.model.add_entries(it, entries)

differences |= self._update_item_state(child)

todo.append(self.model.get_path(child))

for names in allfiles:

entries = [

os.path.join(r, n) for r, n in zip(roots, names)]

child = self.model.add_entries(it, entries)

differences |= self._update_item_state(child)

else:

# Our subtree is empty, or has been filtered to be empty

if (tree.STATE_NORMAL in self.state_filters or

not all(os.path.isdir(f) for f in roots)):

self.model.add_empty(it)

if self.model.iter_parent(it) is None:

expanded.add(tree_path_as_tuple(rootpath))

else:

# At this point, we have an empty folder tree node; we can

# prune this and any ancestors that then end up empty.

while not self.model.iter_has_child(it):

parent = self.model.iter_parent(it)

# In our tree, there is always a top-level parent with

# no siblings. If we're here, we have an empty tree.

if parent is None:

self.model.add_empty(it)

break

# Remove the current row, and then revalidate all

# sibling paths on the stack by removing and

# readding them.

had_siblings = self.model.remove(it)

if had_siblings:

parent_path = self.model.get_path(parent)

for path in todo:

if parent_path.is_ancestor(path):

path.prev()

it = parent

if differences:

expanded.add(tree_path_as_tuple(path))

if invalid_filenames or shadowed_entries:

self._show_tree_wide_errors(invalid_filenames, shadowed_entries)

elif rootpath == Gtk.TreePath.new_first() and not expanded:

self._show_identical_status()

self.treeview[0].expand_to_path(Gtk.TreePath(("0",)))

for path in sorted(expanded):

self.treeview[0].expand_to_path(Gtk.TreePath(path))

yield _("[%s] Done") % self.label_text

self._scan_in_progress -= 1

self.force_cursor_recalculate = True

self.treeview[0].set_cursor(Gtk.TreePath.new_first())

def _show_identical_status(self):

primary = _("Folders have no differences")

identical_note = _(

"Contents of scanned files in folders are identical.")

shallow_note = _(

"Scanned files in folders appear identical, but contents have not "

"been scanned.")

file_filter_qualifier = _(

"File filters are in use, so not all files have been scanned.")

text_filter_qualifier = _(

"Text filters are in use and may be masking content differences.")

is_shallow = self.props.shallow_comparison

have_file_filters = any(f.active for f in self.name_filters)

have_text_filters = any(f.active for f in self.text_filters)

secondary = [shallow_note if is_shallow else identical_note]

if have_file_filters:

secondary.append(file_filter_qualifier)

if not is_shallow and have_text_filters:

secondary.append(text_filter_qualifier)

secondary = " ".join(secondary)

for pane in range(self.num_panes):

msgarea = self.msgarea_mgr[pane].new_from_text_and_icon(

'dialog-information-symbolic', primary, secondary)

button = msgarea.add_button(_("Hide"), Gtk.ResponseType.CLOSE)

if pane == 0:

button.props.label = _("Hi_de")

def clear_all(*args):

for p in range(self.num_panes):

self.msgarea_mgr[p].clear()

msgarea.connect("response", clear_all)

msgarea.show_all()

def _show_tree_wide_errors(self, invalid_filenames, shadowed_entries):

header = _("Multiple errors occurred while scanning this folder")

invalid_header = _("Files with invalid encodings found")

# TRANSLATORS: This is followed by a list of files

invalid_secondary = _("Some files were in an incorrect encoding. "

"The names are something like:")

shadowed_header = _("Files hidden by case insensitive comparison")

# TRANSLATORS: This is followed by a list of files

shadowed_secondary = _("You are running a case insensitive comparison "

"on a case sensitive filesystem. The following "

"files in this folder are hidden:")

invalid_entries = [[] for i in range(self.num_panes)]

for pane, root, f in invalid_filenames:

invalid_entries[pane].append(os.path.join(root, f))

formatted_entries = [[] for i in range(self.num_panes)]

for pane, root, f1, f2 in shadowed_entries:

paths = [os.path.join(root, f) for f in (f1, f2)]

entry_str = _("“%s” hidden by “%s”") % (paths[0], paths[1])

formatted_entries[pane].append(entry_str)

if invalid_filenames or shadowed_entries:

for pane in range(self.num_panes):

invalid = "\n".join(invalid_entries[pane])

shadowed = "\n".join(formatted_entries[pane])

if invalid and shadowed:

messages = (invalid_secondary, invalid, "",

shadowed_secondary, shadowed)

elif invalid:

header = invalid_header